From ee7f792425d3673e86fa8f7224cad145efa19d68 Mon Sep 17 00:00:00 2001 From: pancx Date: Wed, 18 Dec 2024 11:47:57 +0800 Subject: [PATCH 01/47] [#5831] fix(CLI): Fix CLi gives unexpected output when setting a tag Running the command gcli tag set --metalake metalake_demo or gcli tag remove --metalake metalake_demo gives unexpected output.it should give some help information. --- .../src/main/java/org/apache/gravitino/cli/ErrorMessages.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java index 323f0fc2aed..fac1744e13a 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java @@ -30,6 +30,8 @@ public class ErrorMessages { public static final String UNKNOWN_TABLE = "Unknown table name."; public static final String MALFORMED_NAME = "Malformed entity name."; public static final String MISSING_NAME = "Missing --name option."; + public static final String MISSING_ENTITY = + "Unable to determine the entity. Please specify one" + " using the --name option."; public static final String METALAKE_EXISTS = "Metalake already exists."; public static final String CATALOG_EXISTS = "Catalog already exists."; public static final String SCHEMA_EXISTS = "Schema already exists."; From e9d6f0cc9f1bd8510a5e00ba0f9e3c19b40879ef Mon Sep 17 00:00:00 2001 From: pancx Date: Wed, 18 Dec 2024 18:24:48 +0800 Subject: [PATCH 02/47] [#5831] fix(CLI): Fix CLi gives unexpected output when setting a tag fix failed test case. --- .../src/test/java/org/apache/gravitino/cli/TestTagCommands.java | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java index 58beb02a8d8..b68d6c81050 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java @@ -315,6 +315,7 @@ void testDeleteAllTagCommand() { when(mockCommandLine.hasOption(GravitinoOptions.TAG)).thenReturn(false); when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); when(mockCommandLine.hasOption(GravitinoOptions.FORCE)).thenReturn(true); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema.table"); GravitinoCommandLine commandLine = spy( From e7f16d3df06b5654901dabc1bfec90af13110399 Mon Sep 17 00:00:00 2001 From: pancx Date: Wed, 18 Dec 2024 18:30:41 +0800 Subject: [PATCH 03/47] [#5831] fix(CLI): Fix CLi gives unexpected output when setting a tag fix some problems according to reviewer. --- .../src/main/java/org/apache/gravitino/cli/ErrorMessages.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java index fac1744e13a..323f0fc2aed 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java @@ -30,8 +30,6 @@ public class ErrorMessages { public static final String UNKNOWN_TABLE = "Unknown table name."; public static final String MALFORMED_NAME = "Malformed entity name."; public static final String MISSING_NAME = "Missing --name option."; - public static final String MISSING_ENTITY = - "Unable to determine the entity. Please specify one" + " using the --name option."; public static final String METALAKE_EXISTS = "Metalake already exists."; public static final String CATALOG_EXISTS = "Catalog already exists."; public static final String SCHEMA_EXISTS = "Schema already exists."; From d4353553e91f5e7ebbe89d7948d7b031b68964ed Mon Sep 17 00:00:00 2001 From: pancx Date: Thu, 19 Dec 2024 13:50:42 +0800 Subject: [PATCH 04/47] [#5831] fix(CLI): Fix CLi gives unexpected output when setting a tag Change the code due to new pr merged. --- .../gravitino/cli/GravitinoCommandLine.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 3603a230f28..ef3ea658c98 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -619,7 +619,14 @@ protected void handleTagCommand() { if (propertySet != null && valueSet != null) { newSetTagProperty(url, ignore, metalake, getOneTag(tags), propertySet, valueSet).handle(); } else if (propertySet == null && valueSet == null) { + if (!hasEntity(name)) { + System.err.println(ErrorMessages.MALFORMED_NAME); + return; + } newTagEntity(url, ignore, metalake, name, tags).handle(); + } else { + System.err.println( + "This command cannot be executed. The tag set command only supports configuring tag properties or attaching tags to entities."); } break; @@ -627,12 +634,20 @@ protected void handleTagCommand() { boolean isTag = line.hasOption(GravitinoOptions.TAG); if (!isTag) { boolean forceRemove = line.hasOption(GravitinoOptions.FORCE); + if (!hasEntity(name)) { + System.err.println(ErrorMessages.MALFORMED_NAME); + return; + } newRemoveAllTags(url, ignore, metalake, name, forceRemove).handle(); } else { String propertyRemove = line.getOptionValue(GravitinoOptions.PROPERTY); if (propertyRemove != null) { newRemoveTagProperty(url, ignore, metalake, getOneTag(tags), propertyRemove).handle(); } else { + if (!hasEntity(name)) { + System.err.println(ErrorMessages.MALFORMED_NAME); + return; + } newUntagEntity(url, ignore, metalake, name, tags).handle(); } } @@ -664,6 +679,13 @@ private String getOneTag(String[] tags) { return tags[0]; } + private boolean hasEntity(FullName name) { + // TODO fileset and topic + return !(Objects.isNull(name.getCatalogName()) + && Objects.isNull(name.getSchemaName()) + && Objects.isNull(name.getTableName())); + } + /** Handles the command execution for Roles based on command type and the command line options. */ protected void handleRoleCommand() { String url = getUrl(); From 863e247dc61ae9970d12c6e970e02788c13ea724 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Thu, 19 Dec 2024 23:32:50 +1100 Subject: [PATCH 05/47] [#5162] Add grant and revoke privileges to the Gravitino CLI. (#5783) ### What changes were proposed in this pull request? Add grant and revoke privileges to the Gravitino CLI. ### Why are the changes needed? To complete the role commands. Fix: #5162 ### Does this PR introduce _any_ user-facing change? No but it adds two more commands. ### How was this patch tested? Tested locally. --- .../org/apache/gravitino/cli/FullName.java | 13 ++ .../gravitino/cli/GravitinoCommandLine.java | 9 ++ .../gravitino/cli/GravitinoOptions.java | 2 + .../org/apache/gravitino/cli/Privileges.java | 119 ++++++++++++++++++ .../gravitino/cli/TestableCommandLine.java | 22 ++++ .../cli/commands/GrantPrivilegesToRole.java | 106 ++++++++++++++++ .../cli/commands/MetadataCommand.java | 83 ++++++++++++ .../commands/RevokePrivilegesFromRole.java | 106 ++++++++++++++++ .../gravitino/cli/commands/RoleDetails.java | 13 +- .../apache/gravitino/cli/TestPrivileges.java | 52 ++++++++ .../gravitino/cli/TestRoleCommands.java | 62 +++++++++ docs/cli.md | 19 +++ 12 files changed, 601 insertions(+), 5 deletions(-) create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/Privileges.java create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/commands/GrantPrivilegesToRole.java create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetadataCommand.java create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokePrivilegesFromRole.java create mode 100644 clients/cli/src/test/java/org/apache/gravitino/cli/TestPrivileges.java diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java index c53a5adc88f..46a3bb92dce 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java @@ -133,6 +133,19 @@ public String getColumnName() { return getNamePart(3); } + /** + * Retrieves the name from the command line options. + * + * @return The name, or null if not found. + */ + public String getName() { + if (line.hasOption(GravitinoOptions.NAME)) { + return line.getOptionValue(GravitinoOptions.NAME); + } + + return null; + } + /** * Helper method to retrieve a specific part of the full name based on the position of the part. * diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index ef3ea658c98..e54d9e35427 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -694,6 +694,7 @@ protected void handleRoleCommand() { FullName name = new FullName(line); String metalake = name.getMetalakeName(); String role = line.getOptionValue(GravitinoOptions.ROLE); + String[] privileges = line.getOptionValues(GravitinoOptions.PRIVILEGE); Command.setAuthenticationMode(auth, userName); @@ -719,6 +720,14 @@ protected void handleRoleCommand() { newDeleteRole(url, ignore, forceDelete, metalake, role).handle(); break; + case CommandActions.GRANT: + newGrantPrivilegesToRole(url, ignore, metalake, role, name, privileges).handle(); + break; + + case CommandActions.REVOKE: + newRevokePrivilegesFromRole(url, ignore, metalake, role, name, privileges).handle(); + break; + default: System.err.println(ErrorMessages.UNSUPPORTED_ACTION); break; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java index 91993226b92..a42591026a6 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java @@ -45,6 +45,7 @@ public class GravitinoOptions { public static final String OWNER = "owner"; public static final String PARTITION = "partition"; public static final String POSITION = "position"; + public static final String PRIVILEGE = "privilege"; public static final String PROPERTIES = "properties"; public static final String PROPERTY = "property"; public static final String PROVIDER = "provider"; @@ -105,6 +106,7 @@ public Options options() { // Options that support multiple values options.addOption(createArgsOption("p", PROPERTIES, "property name/value pairs")); options.addOption(createArgsOption("t", TAG, "tag name")); + options.addOption(createArgsOption(null, PRIVILEGE, "privilege(s)")); options.addOption(createArgsOption("r", ROLE, "role name")); // Force delete entities and rename metalake operations diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/Privileges.java b/clients/cli/src/main/java/org/apache/gravitino/cli/Privileges.java new file mode 100644 index 00000000000..9d47d8fc9c8 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/Privileges.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import java.util.HashSet; +import org.apache.gravitino.authorization.Privilege; + +public class Privileges { + public static final String CREATE_CATALOG = "create_catalog"; + public static final String USE_CATALOG = "use_catalog"; + public static final String CREATE_SCHEMA = "create_schema"; + public static final String USE_SCHEMA = "use_schema"; + public static final String CREATE_TABLE = "create_table"; + public static final String MODIFY_TABLE = "modify_table"; + public static final String SELECT_TABLE = "select_table"; + public static final String CREATE_FILESET = "create_fileset"; + public static final String WRITE_FILESET = "write_fileset"; + public static final String READ_FILESET = "read_fileset"; + public static final String CREATE_TOPIC = "create_topic"; + public static final String PRODUCE_TOPIC = "produce_topic"; + public static final String CONSUME_TOPIC = "consume_topic"; + public static final String MANAGE_USERS = "manage_users"; + public static final String CREATE_ROLE = "create_role"; + public static final String MANAGE_GRANTS = "manage_grants"; + + private static final HashSet VALID_PRIVILEGES = new HashSet<>(); + + static { + VALID_PRIVILEGES.add(CREATE_CATALOG); + VALID_PRIVILEGES.add(USE_CATALOG); + VALID_PRIVILEGES.add(CREATE_SCHEMA); + VALID_PRIVILEGES.add(USE_SCHEMA); + VALID_PRIVILEGES.add(CREATE_TABLE); + VALID_PRIVILEGES.add(MODIFY_TABLE); + VALID_PRIVILEGES.add(SELECT_TABLE); + VALID_PRIVILEGES.add(CREATE_FILESET); + VALID_PRIVILEGES.add(WRITE_FILESET); + VALID_PRIVILEGES.add(READ_FILESET); + VALID_PRIVILEGES.add(CREATE_TOPIC); + VALID_PRIVILEGES.add(PRODUCE_TOPIC); + VALID_PRIVILEGES.add(CONSUME_TOPIC); + VALID_PRIVILEGES.add(MANAGE_USERS); + VALID_PRIVILEGES.add(CREATE_ROLE); + VALID_PRIVILEGES.add(MANAGE_GRANTS); + } + + /** + * Checks if a given privilege is a valid one. + * + * @param privilege The privilege to check. + * @return true if the privilege is valid, false otherwise. + */ + public static boolean isValid(String privilege) { + return VALID_PRIVILEGES.contains(privilege); + } + + /** + * Converts a string representation of a privilege to the corresponding {@link Privilege.Name}. + * + * @param privilege the privilege to be converted. + * @return the corresponding {@link Privilege.Name} constant, or nullif the privilege is unknown. + */ + public static Privilege.Name toName(String privilege) { + switch (privilege) { + case CREATE_CATALOG: + return Privilege.Name.CREATE_CATALOG; + case USE_CATALOG: + return Privilege.Name.USE_CATALOG; + case CREATE_SCHEMA: + return Privilege.Name.CREATE_SCHEMA; + case USE_SCHEMA: + return Privilege.Name.USE_SCHEMA; + case CREATE_TABLE: + return Privilege.Name.CREATE_TABLE; + case MODIFY_TABLE: + return Privilege.Name.MODIFY_TABLE; + case SELECT_TABLE: + return Privilege.Name.SELECT_TABLE; + case CREATE_FILESET: + return Privilege.Name.CREATE_FILESET; + case WRITE_FILESET: + return Privilege.Name.WRITE_FILESET; + case READ_FILESET: + return Privilege.Name.READ_FILESET; + case CREATE_TOPIC: + return Privilege.Name.CREATE_TOPIC; + case PRODUCE_TOPIC: + return Privilege.Name.PRODUCE_TOPIC; + case CONSUME_TOPIC: + return Privilege.Name.CONSUME_TOPIC; + case MANAGE_USERS: + return Privilege.Name.MANAGE_USERS; + case CREATE_ROLE: + return Privilege.Name.CREATE_ROLE; + case MANAGE_GRANTS: + return Privilege.Name.MANAGE_GRANTS; + default: + System.err.println("Unknown privilege"); + return null; + } + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java index 21fa65d994d..a997a95cee3 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java @@ -50,6 +50,7 @@ import org.apache.gravitino.cli.commands.DeleteTopic; import org.apache.gravitino.cli.commands.DeleteUser; import org.apache.gravitino.cli.commands.FilesetDetails; +import org.apache.gravitino.cli.commands.GrantPrivilegesToRole; import org.apache.gravitino.cli.commands.GroupAudit; import org.apache.gravitino.cli.commands.GroupDetails; import org.apache.gravitino.cli.commands.ListAllTags; @@ -85,6 +86,7 @@ import org.apache.gravitino.cli.commands.RemoveTableProperty; import org.apache.gravitino.cli.commands.RemoveTagProperty; import org.apache.gravitino.cli.commands.RemoveTopicProperty; +import org.apache.gravitino.cli.commands.RevokePrivilegesFromRole; import org.apache.gravitino.cli.commands.RoleAudit; import org.apache.gravitino.cli.commands.RoleDetails; import org.apache.gravitino.cli.commands.SchemaAudit; @@ -862,4 +864,24 @@ protected CreateTable newCreateTable( String comment) { return new CreateTable(url, ignore, metalake, catalog, schema, table, columnFile, comment); } + + protected GrantPrivilegesToRole newGrantPrivilegesToRole( + String url, + boolean ignore, + String metalake, + String role, + FullName entity, + String[] privileges) { + return new GrantPrivilegesToRole(url, ignore, metalake, role, entity, privileges); + } + + protected RevokePrivilegesFromRole newRevokePrivilegesFromRole( + String url, + boolean ignore, + String metalake, + String role, + FullName entity, + String[] privileges) { + return new RevokePrivilegesFromRole(url, ignore, metalake, role, entity, privileges); + } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GrantPrivilegesToRole.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GrantPrivilegesToRole.java new file mode 100644 index 00000000000..e3c9fa4944e --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GrantPrivilegesToRole.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli.commands; + +import java.util.ArrayList; +import java.util.List; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.cli.ErrorMessages; +import org.apache.gravitino.cli.FullName; +import org.apache.gravitino.cli.Privileges; +import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.dto.authorization.PrivilegeDTO; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NoSuchRoleException; + +/** Grants one or more privileges. */ +public class GrantPrivilegesToRole extends MetadataCommand { + + protected final String metalake; + protected final String role; + protected final FullName entity; + protected final String[] privileges; + + /** + * Grants one or more privileges. + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + * @param metalake The name of the metalake. + * @param role The name of the role. + * @param entity The name of the entity. + * @param privileges The list of privileges. + */ + public GrantPrivilegesToRole( + String url, + boolean ignoreVersions, + String metalake, + String role, + FullName entity, + String[] privileges) { + super(url, ignoreVersions); + this.metalake = metalake; + this.entity = entity; + this.role = role; + this.privileges = privileges; + } + + /** Grants one or more privileges. */ + @Override + public void handle() { + try { + GravitinoClient client = buildClient(metalake); + List privilegesList = new ArrayList<>(); + + for (String privilege : privileges) { + if (!Privileges.isValid(privilege)) { + System.err.println("Unknown privilege " + privilege); + return; + } + PrivilegeDTO privilegeDTO = + PrivilegeDTO.builder() + .withName(Privileges.toName(privilege)) + .withCondition(Privilege.Condition.ALLOW) + .build(); + privilegesList.add(privilegeDTO); + } + + MetadataObject metadataObject = constructMetadataObject(entity, client); + client.grantPrivilegesToRole(role, metadataObject, privilegesList); + } catch (NoSuchMetalakeException err) { + System.err.println(ErrorMessages.UNKNOWN_METALAKE); + return; + } catch (NoSuchRoleException err) { + System.err.println(ErrorMessages.UNKNOWN_ROLE); + return; + } catch (NoSuchMetadataObjectException err) { + System.err.println(ErrorMessages.UNKNOWN_USER); + return; + } catch (Exception exp) { + System.err.println(exp.getMessage()); + return; + } + + String all = String.join(",", privileges); + System.out.println(role + " granted " + all + " on " + entity.getName()); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetadataCommand.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetadataCommand.java new file mode 100644 index 00000000000..3f1e347c1fd --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetadataCommand.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli.commands; + +import org.apache.gravitino.Catalog; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.cli.FullName; +import org.apache.gravitino.client.GravitinoClient; + +public class MetadataCommand extends Command { + + /** + * MetadataCommand constructor. + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + */ + public MetadataCommand(String url, boolean ignoreVersions) { + super(url, ignoreVersions); + } + + /** + * Constructs a {@link MetadataObject} based on the provided client, existing metadata object, and + * entity name. + * + * @param entity The name of the entity. + * @param client The Gravitino client. + * @return A MetadataObject of the appropriate type. + * @throws IllegalArgumentException if the entity type cannot be determined or is unknown. + */ + protected MetadataObject constructMetadataObject(FullName entity, GravitinoClient client) { + + MetadataObject metadataObject; + String name = entity.getName(); + + if (entity.hasColumnName()) { + metadataObject = MetadataObjects.of(null, name, MetadataObject.Type.COLUMN); + } else if (entity.hasTableName()) { + Catalog catalog = client.loadCatalog(entity.getCatalogName()); + Catalog.Type catalogType = catalog.type(); + if (catalogType == Catalog.Type.RELATIONAL) { + metadataObject = MetadataObjects.of(null, name, MetadataObject.Type.TABLE); + } else if (catalogType == Catalog.Type.MESSAGING) { + metadataObject = MetadataObjects.of(null, name, MetadataObject.Type.TOPIC); + } else if (catalogType == Catalog.Type.FILESET) { + metadataObject = MetadataObjects.of(null, name, MetadataObject.Type.FILESET); + } else { + throw new IllegalArgumentException("Unknown entity type: " + name); + } + } else if (entity.hasSchemaName()) { + metadataObject = MetadataObjects.of(null, name, MetadataObject.Type.SCHEMA); + } else if (entity.hasCatalogName()) { + metadataObject = MetadataObjects.of(null, name, MetadataObject.Type.CATALOG); + } else if (entity.getMetalakeName() != null) { + metadataObject = MetadataObjects.of(null, name, MetadataObject.Type.METALAKE); + } else { + throw new IllegalArgumentException("Unknown entity type: " + name); + } + return metadataObject; + } + + /* Do nothing, as parent will override. */ + @Override + public void handle() {} +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokePrivilegesFromRole.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokePrivilegesFromRole.java new file mode 100644 index 00000000000..8077532319e --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RevokePrivilegesFromRole.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli.commands; + +import java.util.ArrayList; +import java.util.List; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.cli.ErrorMessages; +import org.apache.gravitino.cli.FullName; +import org.apache.gravitino.cli.Privileges; +import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.dto.authorization.PrivilegeDTO; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NoSuchRoleException; + +/** Revokes one or more privileges. */ +public class RevokePrivilegesFromRole extends MetadataCommand { + + protected final String metalake; + protected final String role; + protected final FullName entity; + protected final String[] privileges; + + /** + * Revokes one or more privileges. + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + * @param metalake The name of the metalake. + * @param role The name of the role. + * @param entity The name of the entity. + * @param privileges The list of privileges. + */ + public RevokePrivilegesFromRole( + String url, + boolean ignoreVersions, + String metalake, + String role, + FullName entity, + String[] privileges) { + super(url, ignoreVersions); + this.metalake = metalake; + this.entity = entity; + this.role = role; + this.privileges = privileges; + } + + /** Revokes One or more privileges. */ + @Override + public void handle() { + try { + GravitinoClient client = buildClient(metalake); + List privilegesList = new ArrayList<>(); + + for (String privilege : privileges) { + if (!Privileges.isValid(privilege)) { + System.err.println("Unknown privilege " + privilege); + return; + } + PrivilegeDTO privilegeDTO = + PrivilegeDTO.builder() + .withName(Privileges.toName(privilege)) + .withCondition(Privilege.Condition.DENY) + .build(); + privilegesList.add(privilegeDTO); + } + + MetadataObject metadataObject = constructMetadataObject(entity, client); + client.revokePrivilegesFromRole(role, metadataObject, privilegesList); + } catch (NoSuchMetalakeException err) { + System.err.println(ErrorMessages.UNKNOWN_METALAKE); + return; + } catch (NoSuchRoleException err) { + System.err.println(ErrorMessages.UNKNOWN_ROLE); + return; + } catch (NoSuchMetadataObjectException err) { + System.err.println(ErrorMessages.UNKNOWN_USER); + return; + } catch (Exception exp) { + System.err.println(exp.getMessage()); + return; + } + + String all = String.join(",", privileges); + System.out.println(role + " revoked " + all + " on " + entity.getName()); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RoleDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RoleDetails.java index 613ee60d2ce..2c1613eded7 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RoleDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RoleDetails.java @@ -20,7 +20,7 @@ package org.apache.gravitino.cli.commands; import java.util.List; -import java.util.stream.Collectors; +import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.cli.ErrorMessages; import org.apache.gravitino.client.GravitinoClient; @@ -65,9 +65,12 @@ public void handle() { return; } - // TODO expand in securable objects PR - String all = objects.stream().map(SecurableObject::name).collect(Collectors.joining(",")); - - System.out.println(all.toString()); + for (SecurableObject object : objects) { + System.out.print(object.name() + "," + object.type() + ","); + for (Privilege privilege : object.privileges()) { + System.out.print(privilege.simpleString() + " "); + } + } + System.out.println(""); } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestPrivileges.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestPrivileges.java new file mode 100644 index 00000000000..b6d39caded5 --- /dev/null +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestPrivileges.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class TestsPrivileges { + + @Test + void testValidPrivilege() { + assertTrue(Privileges.isValid(Privileges.CREATE_CATALOG)); + assertTrue(Privileges.isValid(Privileges.CREATE_TABLE)); + assertTrue(Privileges.isValid(Privileges.CONSUME_TOPIC)); + assertTrue(Privileges.isValid(Privileges.MANAGE_GRANTS)); + } + + @Test + void testInvalidPrivilege() { + assertFalse(Privileges.isValid("non_existent_privilege")); + assertFalse(Privileges.isValid("create_database")); + } + + @Test + void testNullPrivilege() { + assertFalse(Privileges.isValid(null)); + } + + @Test + void testEmptyPrivilege() { + assertFalse(Privileges.isValid("")); + } +} diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java index 179dba14fe8..88b380d63ee 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java @@ -19,7 +19,9 @@ package org.apache.gravitino.cli; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; @@ -29,7 +31,9 @@ import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateRole; import org.apache.gravitino.cli.commands.DeleteRole; +import org.apache.gravitino.cli.commands.GrantPrivilegesToRole; import org.apache.gravitino.cli.commands.ListRoles; +import org.apache.gravitino.cli.commands.RevokePrivilegesFromRole; import org.apache.gravitino.cli.commands.RoleAudit; import org.apache.gravitino.cli.commands.RoleDetails; import org.junit.jupiter.api.BeforeEach; @@ -152,4 +156,62 @@ void testDeleteRoleForceCommand() { commandLine.handleCommandLine(); verify(mockDelete).handle(); } + + @Test + void testGrantPrivilegesToRole() { + GrantPrivilegesToRole mockGrant = mock(GrantPrivilegesToRole.class); + String[] privileges = {"create_table", "modify_table"}; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("admin"); + when(mockCommandLine.hasOption(GravitinoOptions.PRIVILEGE)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.PRIVILEGE)).thenReturn(privileges); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.GRANT)); + doReturn(mockGrant) + .when(commandLine) + .newGrantPrivilegesToRole( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq("admin"), + any(), + eq(privileges)); + commandLine.handleCommandLine(); + verify(mockGrant).handle(); + } + + @Test + void testRevokePrivilegesFromRole() { + RevokePrivilegesFromRole mockRevoke = mock(RevokePrivilegesFromRole.class); + String[] privileges = {"create_table", "modify_table"}; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("admin"); + when(mockCommandLine.hasOption(GravitinoOptions.PRIVILEGE)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.PRIVILEGE)).thenReturn(privileges); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.REVOKE)); + doReturn(mockRevoke) + .when(commandLine) + .newRevokePrivilegesFromRole( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq("admin"), + any(), + eq(privileges)); + commandLine.handleCommandLine(); + verify(mockRevoke).handle(); + } } diff --git a/docs/cli.md b/docs/cli.md index b01ea477562..e6e2f5aa609 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -672,6 +672,12 @@ gcli catalog set --owner --group groupA --name postgres ### Role commands +When granting or revoking privileges the following privileges can be used. + +create_catalog, use_catalog, create_schema, use_schema, create_table, modify_table, select_table, create_fileset, write_fileset, read_fileset, create_topic, produce_topic, consume_topic, manage_users, create_role, manage_grants + +Note that some are only valid for certain entities. + #### Display role details ```bash @@ -721,10 +727,23 @@ gcli group grant --group groupA --role admin ``` #### Remove a role from a group + ```bash gcli group revoke --group groupA --role admin ``` +### Grant a privilege + +```bash +gcli role grant --name catalog_postgres --role admin --privilege create_table modify_table +``` + +### Revoke a privilege + +```bash +gcli role revoke --metalake metalake_demo --name catalog_postgres --role admin --privilege create_table modify_table +``` + ### Topic commands #### Display a topic's details From d5d5a9dda6713394e3e64137b076e9dd237d3a82 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Fri, 20 Dec 2024 01:07:24 +1100 Subject: [PATCH 06/47] [#5896] Exit with -1 when an error occurs in the Gravitino CLI (#5903) ### What changes were proposed in this pull request? Changed to exit with -1 if an error occurs. Not that testing code that calls System.exit can be difficult, so during testing it thows an exception and doesn't exit the VM. Testing also uncovered one bug in updating column default values and this was fixed. ### Why are the changes needed? So scripts can test if the gcli command works. Fix: #5896 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Tested locally. --------- Co-authored-by: Shaofeng Shi --- .../gravitino/cli/GravitinoCommandLine.java | 44 ++++++++++++------- .../apache/gravitino/cli/GravitinoConfig.java | 1 + .../java/org/apache/gravitino/cli/Main.java | 27 ++++++++++-- .../apache/gravitino/cli/ReadTableCSV.java | 1 + .../gravitino/cli/TestableCommandLine.java | 8 ++-- .../gravitino/cli/commands/AddColumn.java | 15 +++---- .../cli/commands/AddRoleToGroup.java | 12 ++--- .../gravitino/cli/commands/AddRoleToUser.java | 12 ++--- .../cli/commands/AllMetalakeDetails.java | 3 +- .../gravitino/cli/commands/CatalogAudit.java | 11 ++--- .../cli/commands/CatalogDetails.java | 6 +-- .../gravitino/cli/commands/ClientVersion.java | 3 +- .../gravitino/cli/commands/Command.java | 13 +++++- .../gravitino/cli/commands/CreateCatalog.java | 9 ++-- .../gravitino/cli/commands/CreateFileset.java | 15 +++---- .../gravitino/cli/commands/CreateGroup.java | 9 ++-- .../cli/commands/CreateMetalake.java | 6 +-- .../gravitino/cli/commands/CreateRole.java | 9 ++-- .../gravitino/cli/commands/CreateSchema.java | 12 ++--- .../gravitino/cli/commands/CreateTable.java | 27 +++++------- .../gravitino/cli/commands/CreateTag.java | 18 +++----- .../gravitino/cli/commands/CreateTopic.java | 12 ++--- .../gravitino/cli/commands/CreateUser.java | 9 ++-- .../gravitino/cli/commands/DeleteCatalog.java | 9 ++-- .../gravitino/cli/commands/DeleteColumn.java | 18 +++----- .../gravitino/cli/commands/DeleteFileset.java | 15 +++---- .../gravitino/cli/commands/DeleteGroup.java | 9 ++-- .../cli/commands/DeleteMetalake.java | 6 +-- .../gravitino/cli/commands/DeleteRole.java | 9 ++-- .../gravitino/cli/commands/DeleteSchema.java | 12 ++--- .../gravitino/cli/commands/DeleteTable.java | 15 +++---- .../gravitino/cli/commands/DeleteTag.java | 18 +++----- .../gravitino/cli/commands/DeleteTopic.java | 12 ++--- .../gravitino/cli/commands/DeleteUser.java | 9 ++-- .../cli/commands/FilesetDetails.java | 15 +++---- .../gravitino/cli/commands/GroupDetails.java | 9 ++-- .../gravitino/cli/commands/ListAllTags.java | 6 +-- .../cli/commands/ListCatalogProperties.java | 9 ++-- .../gravitino/cli/commands/ListCatalogs.java | 4 +- .../gravitino/cli/commands/ListColumns.java | 3 +- .../cli/commands/ListEntityTags.java | 15 +++---- .../cli/commands/ListFilesetProperties.java | 12 ++--- .../gravitino/cli/commands/ListFilesets.java | 12 ++--- .../gravitino/cli/commands/ListGroups.java | 6 +-- .../gravitino/cli/commands/ListIndexes.java | 5 +-- .../cli/commands/ListMetalakeProperties.java | 6 +-- .../gravitino/cli/commands/ListMetalakes.java | 2 +- .../gravitino/cli/commands/ListRoles.java | 6 +-- .../gravitino/cli/commands/ListSchema.java | 9 ++-- .../cli/commands/ListSchemaProperties.java | 12 ++--- .../cli/commands/ListTableProperties.java | 15 +++---- .../gravitino/cli/commands/ListTables.java | 3 +- .../cli/commands/ListTagProperties.java | 9 ++-- .../cli/commands/ListTopicProperties.java | 15 +++---- .../gravitino/cli/commands/ListTopics.java | 6 +-- .../gravitino/cli/commands/ListUsers.java | 6 +-- .../gravitino/cli/commands/MetalakeAudit.java | 8 ++-- .../cli/commands/MetalakeDetails.java | 4 +- .../gravitino/cli/commands/OwnerDetails.java | 9 ++-- .../gravitino/cli/commands/RemoveAllTags.java | 15 +++---- .../cli/commands/RemoveCatalogProperty.java | 9 ++-- .../cli/commands/RemoveFilesetProperty.java | 15 +++---- .../cli/commands/RemoveMetalakeProperty.java | 6 +-- .../cli/commands/RemoveRoleFromGroup.java | 12 ++--- .../cli/commands/RemoveRoleFromUser.java | 12 ++--- .../cli/commands/RemoveSchemaProperty.java | 12 ++--- .../cli/commands/RemoveTableProperty.java | 15 +++---- .../cli/commands/RemoveTagProperty.java | 9 ++-- .../cli/commands/RemoveTopicProperty.java | 15 +++---- .../gravitino/cli/commands/RoleDetails.java | 9 ++-- .../gravitino/cli/commands/SchemaAudit.java | 14 +++--- .../gravitino/cli/commands/SchemaDetails.java | 12 ++--- .../gravitino/cli/commands/ServerVersion.java | 3 +- .../cli/commands/SetCatalogProperty.java | 9 ++-- .../cli/commands/SetFilesetProperty.java | 15 +++---- .../cli/commands/SetMetalakeProperty.java | 6 +-- .../gravitino/cli/commands/SetOwner.java | 9 ++-- .../cli/commands/SetSchemaProperty.java | 12 ++--- .../cli/commands/SetTableProperty.java | 15 +++---- .../cli/commands/SetTagProperty.java | 9 ++-- .../cli/commands/SetTopicProperty.java | 15 +++---- .../gravitino/cli/commands/TableAudit.java | 5 +-- .../gravitino/cli/commands/TableCommand.java | 10 ++--- .../gravitino/cli/commands/TableDetails.java | 3 +- .../cli/commands/TableDistribution.java | 5 +-- .../cli/commands/TablePartition.java | 5 +-- .../cli/commands/TableSortOrder.java | 5 +-- .../gravitino/cli/commands/TagDetails.java | 9 ++-- .../gravitino/cli/commands/TagEntity.java | 15 +++---- .../gravitino/cli/commands/TopicDetails.java | 15 +++---- .../gravitino/cli/commands/UntagEntity.java | 15 +++---- .../cli/commands/UpdateCatalogComment.java | 9 ++-- .../cli/commands/UpdateCatalogName.java | 9 ++-- .../commands/UpdateColumnAutoIncrement.java | 18 +++----- .../cli/commands/UpdateColumnComment.java | 18 +++----- .../cli/commands/UpdateColumnDatatype.java | 18 +++----- .../cli/commands/UpdateColumnDefault.java | 18 +++----- .../cli/commands/UpdateColumnName.java | 18 +++----- .../cli/commands/UpdateColumnNullability.java | 18 +++----- .../cli/commands/UpdateColumnPosition.java | 18 +++----- .../cli/commands/UpdateFilesetComment.java | 15 +++---- .../cli/commands/UpdateFilesetName.java | 15 +++---- .../cli/commands/UpdateMetalakeComment.java | 6 +-- .../cli/commands/UpdateMetalakeName.java | 6 +-- .../cli/commands/UpdateTableComment.java | 15 +++---- .../cli/commands/UpdateTableName.java | 15 +++---- .../cli/commands/UpdateTagComment.java | 9 ++-- .../gravitino/cli/commands/UpdateTagName.java | 9 ++-- .../cli/commands/UpdateTopicComment.java | 15 +++---- .../gravitino/cli/commands/UserDetails.java | 9 ++-- .../gravitino/cli/TestColumnCommands.java | 1 - .../gravitino/cli/TestGroupCommands.java | 10 +++-- .../org/apache/gravitino/cli/TestMain.java | 8 +++- .../gravitino/cli/TestUserCommands.java | 10 +++-- 114 files changed, 471 insertions(+), 771 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index e54d9e35427..4cbfaacc5ad 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -127,7 +127,7 @@ public static void displayHelp(Options options) { /** Executes the appropriate command based on the command type. */ private void executeCommand() { - if (command.equals(CommandActions.HELP)) { + if (CommandActions.HELP.equals(command)) { handleHelpCommand(); } else if (line.hasOption(GravitinoOptions.OWNER)) { handleOwnerCommand(); @@ -185,7 +185,7 @@ private void handleMetalakeCommand() { case CommandActions.CREATE: if (Objects.isNull(metalake)) { System.err.println(CommandEntities.METALAKE + " is not defined"); - return; + Main.exit(-1); } String comment = line.getOptionValue(GravitinoOptions.COMMENT); newCreateMetalake(url, ignore, metalake, comment).handle(); @@ -225,6 +225,7 @@ private void handleMetalakeCommand() { default: System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); break; } } @@ -300,6 +301,7 @@ private void handleCatalogCommand() { default: System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); break; } } @@ -361,6 +363,7 @@ private void handleSchemaCommand() { default: System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); break; } } @@ -391,7 +394,7 @@ private void handleTableCommand() { if (!missingEntities.isEmpty()) { System.err.println( "Missing required argument(s): " + Joiner.on(", ").join(missingEntities)); - return; + Main.exit(-1); } newListTables(url, ignore, metalake, catalog, schema).handle(); @@ -516,6 +519,7 @@ protected void handleUserCommand() { default: System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); break; } } @@ -571,6 +575,7 @@ protected void handleGroupCommand() { default: System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); break; } } @@ -670,6 +675,7 @@ protected void handleTagCommand() { default: System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); break; } } @@ -730,6 +736,7 @@ protected void handleRoleCommand() { default: System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); break; } } @@ -810,7 +817,8 @@ private void handleColumnCommand() { newUpdateColumnName(url, ignore, metalake, catalog, schema, table, column, newName) .handle(); } - if (line.hasOption(GravitinoOptions.DATATYPE)) { + if (line.hasOption(GravitinoOptions.DATATYPE) + && !line.hasOption(GravitinoOptions.DEFAULT)) { String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); newUpdateColumnDatatype(url, ignore, metalake, catalog, schema, table, column, datatype) .handle(); @@ -844,6 +852,7 @@ private void handleColumnCommand() { default: System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); break; } } @@ -859,9 +868,10 @@ private void handleHelpCommand() { while ((helpLine = reader.readLine()) != null) { helpMessage.append(helpLine).append(System.lineSeparator()); } - System.err.print(helpMessage.toString()); + System.out.print(helpMessage.toString()); } catch (IOException e) { System.err.println("Failed to load help message: " + e.getMessage()); + Main.exit(-1); } } @@ -915,15 +925,17 @@ private void handleTopicCommand() { String metalake = name.getMetalakeName(); String catalog = name.getCatalogName(); String schema = name.getSchemaName(); - String topic = name.getTopicName(); Command.setAuthenticationMode(auth, userName); - switch (command) { - case CommandActions.LIST: - newListTopics(url, ignore, metalake, catalog, schema).handle(); - break; + if (CommandActions.LIST.equals(command)) { + newListTopics(url, ignore, metalake, catalog, schema).handle(); + return; + } + String topic = name.getTopicName(); + + switch (command) { case CommandActions.DETAILS: newTopicDetails(url, ignore, metalake, catalog, schema, topic).handle(); break; @@ -988,19 +1000,21 @@ private void handleFilesetCommand() { String metalake = name.getMetalakeName(); String catalog = name.getCatalogName(); String schema = name.getSchemaName(); - String fileset = name.getFilesetName(); Command.setAuthenticationMode(auth, userName); + if (CommandActions.LIST.equals(command)) { + newListFilesets(url, ignore, metalake, catalog, schema).handle(); + return; + } + + String fileset = name.getFilesetName(); + switch (command) { case CommandActions.DETAILS: newFilesetDetails(url, ignore, metalake, catalog, schema, fileset).handle(); break; - case CommandActions.LIST: - newListFilesets(url, ignore, metalake, catalog, schema).handle(); - break; - case CommandActions.CREATE: { String comment = line.getOptionValue(GravitinoOptions.COMMENT); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoConfig.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoConfig.java index bb9aa5312e7..37f52814e05 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoConfig.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoConfig.java @@ -82,6 +82,7 @@ public void read() { return; } catch (IOException exp) { System.err.println(exp.getMessage()); + Main.exit(-1); } if (prop.containsKey(metalakeKey)) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/Main.java b/clients/cli/src/main/java/org/apache/gravitino/cli/Main.java index 4707da16d21..1f4a3926ef5 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/Main.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/Main.java @@ -29,6 +29,8 @@ /* Entry point for the Gravitino command line. */ public class Main { + public static boolean useExit = true; + public static void main(String[] args) { CommandLineParser parser = new DefaultParser(); Options options = new GravitinoOptions().options(); @@ -43,7 +45,7 @@ public static void main(String[] args) { String[] extra = line.getArgs(); if (extra.length > 2) { System.err.println(ErrorMessages.TOO_MANY_ARGUMENTS); - return; + exit(-1); } String command = resolveCommand(line); GravitinoCommandLine commandLine = new GravitinoCommandLine(line, options, entity, command); @@ -56,6 +58,24 @@ public static void main(String[] args) { } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); GravitinoCommandLine.displayHelp(options); + exit(-1); + } + } + + /** + * Exits the application with the specified exit code. + * + *

If the {@code useExit} flag is set to {@code true}, the application will terminate using + * {@link System#exit(int)}. Otherwise, a {@link RuntimeException} is thrown with a message + * containing the exit code. Helps with testing. + * + * @param code the exit code to use when exiting the application + */ + public static void exit(int code) { + if (useExit) { + System.exit(code); + } else { + throw new RuntimeException("Exit with code " + code); } } @@ -83,7 +103,8 @@ protected static String resolveCommand(CommandLine line) { } System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); - return null; + exit(-1); + return null; // not needed but gives error if not here } /** @@ -102,7 +123,7 @@ protected static String resolveEntity(CommandLine line) { return entity; } else { System.err.println(ErrorMessages.UNKNOWN_ENTITY); - return null; + exit(-1); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ReadTableCSV.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ReadTableCSV.java index 3a9ca3a6dea..dc9f3a9f2e6 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/ReadTableCSV.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ReadTableCSV.java @@ -145,6 +145,7 @@ public Map> parse(String csvFile) { } } catch (IOException exp) { System.err.println(exp.getMessage()); + Main.exit(-1); } return tableData; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java index a997a95cee3..41909f7209e 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java @@ -443,13 +443,13 @@ protected DeleteGroup newDeleteGroup( } protected RemoveRoleFromGroup newRemoveRoleFromGroup( - String url, boolean ignore, String metalake, String user, String role) { - return new RemoveRoleFromGroup(url, ignore, metalake, user, role); + String url, boolean ignore, String metalake, String group, String role) { + return new RemoveRoleFromGroup(url, ignore, metalake, group, role); } protected AddRoleToGroup newAddRoleToGroup( - String url, boolean ignore, String metalake, String user, String role) { - return new AddRoleToGroup(url, ignore, metalake, user, role); + String url, boolean ignore, String metalake, String group, String role) { + return new AddRoleToGroup(url, ignore, metalake, group, role); } protected RoleDetails newRoleDetails(String url, boolean ignore, String metalake, String role) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddColumn.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddColumn.java index 39dc6eb9d65..71603607166 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddColumn.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddColumn.java @@ -112,20 +112,15 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " added to table " + table + "."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddRoleToGroup.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddRoleToGroup.java index c210edbe5c7..ebe3ea15200 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddRoleToGroup.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddRoleToGroup.java @@ -59,17 +59,13 @@ public void handle() { roles.add(role); client.grantRolesToGroup(roles, group); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchRoleException err) { - System.err.println(ErrorMessages.UNKNOWN_ROLE); - return; + exitWithError(ErrorMessages.UNKNOWN_ROLE); } catch (NoSuchUserException err) { - System.err.println(ErrorMessages.UNKNOWN_USER); - return; + exitWithError(ErrorMessages.UNKNOWN_USER); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(role + " added to " + group); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddRoleToUser.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddRoleToUser.java index 7261217ffdb..66985b7dfb8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddRoleToUser.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AddRoleToUser.java @@ -59,17 +59,13 @@ public void handle() { roles.add(role); client.grantRolesToUser(roles, user); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchRoleException err) { - System.err.println(ErrorMessages.UNKNOWN_ROLE); - return; + exitWithError(ErrorMessages.UNKNOWN_ROLE); } catch (NoSuchUserException err) { - System.err.println(ErrorMessages.UNKNOWN_USER); - return; + exitWithError(ErrorMessages.UNKNOWN_USER); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(role + " added to " + user); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AllMetalakeDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AllMetalakeDetails.java index 61df5facda6..07d61dcaa7c 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AllMetalakeDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AllMetalakeDetails.java @@ -45,8 +45,7 @@ public void handle() { GravitinoAdminClient client = buildAdminClient(); metalakes = client.listMetalakes(); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } List metalakeDetails = new ArrayList<>(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogAudit.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogAudit.java index 588d8bf4217..f1ea8ac7b52 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogAudit.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogAudit.java @@ -47,19 +47,16 @@ public CatalogAudit(String url, boolean ignoreVersions, String metalake, String /** Displays the audit information of a specified catalog. */ @Override public void handle() { - Catalog result; + Catalog result = null; try (GravitinoClient client = buildClient(metalake)) { result = client.loadCatalog(this.catalog); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (result != null) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogDetails.java index 3f0e758f47e..a204f560d09 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogDetails.java @@ -57,11 +57,11 @@ public void handle() { result = client.loadCatalog(catalog); output(result); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); + exitWithError(exp.getMessage()); } } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ClientVersion.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ClientVersion.java index 13c95d503c5..6bc2200b7b2 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ClientVersion.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ClientVersion.java @@ -42,8 +42,7 @@ public void handle() { GravitinoAdminClient client = buildAdminClient(); version = client.clientVersion().version(); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println("Apache Gravitino " + version); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/Command.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/Command.java index 59b1b90246f..f91dae40425 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/Command.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/Command.java @@ -24,6 +24,7 @@ import java.io.File; import org.apache.gravitino.cli.GravitinoConfig; import org.apache.gravitino.cli.KerberosData; +import org.apache.gravitino.cli.Main; import org.apache.gravitino.cli.OAuthData; import org.apache.gravitino.cli.outputs.PlainFormat; import org.apache.gravitino.cli.outputs.TableFormat; @@ -75,6 +76,16 @@ public Command(String url, boolean ignoreVersions, String outputFormat) { this.outputFormat = outputFormat; } + /** + * Prints an error message and exits with a non-zero status. + * + * @param error The error message to display before exiting. + */ + public void exitWithError(String error) { + System.err.println(error); + Main.exit(-1); + } + /** * Sets the authentication mode and user credentials for the command. * @@ -154,7 +165,7 @@ protected Builder constructClient(Builder builder = builder.withKerberosAuth(tokenProvider); } else { - System.err.println("Unsupported authentication type " + authentication); + exitWithError("Unsupported authentication type " + authentication); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateCatalog.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateCatalog.java index 7adf16db7f2..e0c11c1e040 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateCatalog.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateCatalog.java @@ -72,14 +72,11 @@ public void handle() { comment, properties); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.METALAKE_EXISTS); - return; + exitWithError(ErrorMessages.METALAKE_EXISTS); } catch (CatalogAlreadyExistsException err) { - System.err.println(ErrorMessages.CATALOG_EXISTS); - return; + exitWithError(ErrorMessages.CATALOG_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(catalog + " catalog created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateFileset.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateFileset.java index bc109fcc52b..1abcb5d3cbe 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateFileset.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateFileset.java @@ -82,20 +82,15 @@ public void handle() { .asFilesetCatalog() .createFileset(name, comment, filesetType, location, null); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (FilesetAlreadyExistsException err) { - System.err.println(ErrorMessages.FILESET_EXISTS); - return; + exitWithError(ErrorMessages.FILESET_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(fileset + " created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateGroup.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateGroup.java index 8f9fa985d6d..8afd35c6403 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateGroup.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateGroup.java @@ -49,14 +49,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); client.addGroup(group); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (GroupAlreadyExistsException err) { - System.err.println(ErrorMessages.GROUP_EXISTS); - return; + exitWithError(ErrorMessages.GROUP_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(group + " created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateMetalake.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateMetalake.java index 1c8f9f353bf..9a3c033f028 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateMetalake.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateMetalake.java @@ -48,11 +48,9 @@ public void handle() { GravitinoAdminClient client = buildAdminClient(); client.createMetalake(metalake, comment, null); } catch (MetalakeAlreadyExistsException err) { - System.err.println(ErrorMessages.METALAKE_EXISTS); - return; + exitWithError(ErrorMessages.METALAKE_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(metalake + " created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateRole.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateRole.java index d3b71f45aaa..fea6fe4a720 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateRole.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateRole.java @@ -50,14 +50,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); client.createRole(role, null, Collections.EMPTY_LIST); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (RoleAlreadyExistsException err) { - System.err.println(ErrorMessages.ROLE_EXISTS); - return; + exitWithError(ErrorMessages.ROLE_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(role + " created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateSchema.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateSchema.java index 41f3df93a9f..34243c1e4ed 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateSchema.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateSchema.java @@ -62,17 +62,13 @@ public void handle() { GravitinoClient client = buildClient(metalake); client.loadCatalog(catalog).asSchemas().createSchema(schema, comment, null); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (SchemaAlreadyExistsException err) { - System.err.println(ErrorMessages.SCHEMA_EXISTS); - return; + exitWithError(ErrorMessages.SCHEMA_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(schema + " created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTable.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTable.java index 91e7b6e3046..fefa6267221 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTable.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTable.java @@ -72,45 +72,38 @@ public CreateTable( /** Create a new table. */ @Override public void handle() { - NameIdentifier tableName; - GravitinoClient client; + NameIdentifier tableName = null; + GravitinoClient client = null; ReadTableCSV readTableCSV = new ReadTableCSV(); Map> tableData; - Column[] columns; + Column[] columns = {}; try { tableName = NameIdentifier.of(schema, table); client = buildClient(metalake); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println("Error initializing client or table name: " + exp.getMessage()); - return; + exitWithError("Error initializing client or table name: " + exp.getMessage()); } try { tableData = readTableCSV.parse(columnFile); columns = readTableCSV.columns(tableData); } catch (Exception exp) { - System.err.println("Error reading or parsing column file: " + exp.getMessage()); - return; + exitWithError("Error reading or parsing column file: " + exp.getMessage()); } try { client.loadCatalog(catalog).asTableCatalog().createTable(tableName, columns, comment, null); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (TableAlreadyExistsException err) { - System.err.println(ErrorMessages.TABLE_EXISTS); - return; + exitWithError(ErrorMessages.TABLE_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(table + " created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTag.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTag.java index 373bf0db7be..0dd4289bb75 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTag.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTag.java @@ -69,14 +69,11 @@ private void handleOnlyOneTag() { GravitinoClient client = buildClient(metalake); client.createTag(tags[0], comment, null); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (TagAlreadyExistsException err) { - System.err.println(ErrorMessages.TAG_EXISTS); - return; + exitWithError(ErrorMessages.TAG_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println("Tag " + tags[0] + " created"); @@ -91,14 +88,11 @@ private void handleMultipleTags() { created.add(tag); } } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (TagAlreadyExistsException err) { - System.err.println(ErrorMessages.TAG_EXISTS); - return; + exitWithError(ErrorMessages.TAG_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (!created.isEmpty()) { System.out.println("Tags " + String.join(",", created) + " created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTopic.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTopic.java index 955d1e28911..61d3db4472f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTopic.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateTopic.java @@ -71,17 +71,13 @@ public void handle() { GravitinoClient client = buildClient(metalake); client.loadCatalog(catalog).asTopicCatalog().createTopic(name, comment, null, null); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (TopicAlreadyExistsException err) { - System.err.println(ErrorMessages.TOPIC_EXISTS); - return; + exitWithError(ErrorMessages.TOPIC_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(topic + " topic created."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateUser.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateUser.java index 150bf64ce01..6f786778e17 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateUser.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateUser.java @@ -49,14 +49,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); client.addUser(user); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (UserAlreadyExistsException err) { - System.err.println(ErrorMessages.USER_EXISTS); - return; + exitWithError(ErrorMessages.USER_EXISTS); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(user + " created"); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteCatalog.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteCatalog.java index 65ebde4e354..6c5fbaee97d 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteCatalog.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteCatalog.java @@ -61,14 +61,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); deleted = client.dropCatalog(catalog); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteColumn.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteColumn.java index 2e2bbd3f96c..17ac22d1284 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteColumn.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteColumn.java @@ -75,23 +75,17 @@ public void handle() { NameIdentifier name = NameIdentifier.of(schema, table); client.loadCatalog(catalog).asTableCatalog().alterTable(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchColumnException err) { - System.err.println(ErrorMessages.UNKNOWN_COLUMN); - return; + exitWithError(ErrorMessages.UNKNOWN_COLUMN); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " deleted."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteFileset.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteFileset.java index bc76dcb2696..2a818f64b3b 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteFileset.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteFileset.java @@ -77,20 +77,15 @@ public void handle() { GravitinoClient client = buildClient(metalake); deleted = client.loadCatalog(catalog).asFilesetCatalog().dropFileset(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchFilesetException err) { - System.err.println(ErrorMessages.UNKNOWN_FILESET); - return; + exitWithError(ErrorMessages.UNKNOWN_FILESET); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteGroup.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteGroup.java index 3c3689dc371..641b43ddac9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteGroup.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteGroup.java @@ -61,14 +61,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); deleted = client.removeGroup(group); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchGroupException err) { - System.err.println(ErrorMessages.UNKNOWN_GROUP); - return; + exitWithError(ErrorMessages.UNKNOWN_GROUP); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteMetalake.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteMetalake.java index 2162d181837..386dde92130 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteMetalake.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteMetalake.java @@ -55,11 +55,9 @@ public void handle() { GravitinoAdminClient client = buildAdminClient(); deleted = client.dropMetalake(metalake); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteRole.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteRole.java index 0338c0c370f..f175d95043f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteRole.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteRole.java @@ -61,14 +61,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); deleted = client.deleteRole(role); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchRoleException err) { - System.err.println(ErrorMessages.UNKNOWN_ROLE); - return; + exitWithError(ErrorMessages.UNKNOWN_ROLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteSchema.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteSchema.java index e1676a076c1..826bdba2fb5 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteSchema.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteSchema.java @@ -70,17 +70,13 @@ public void handle() { GravitinoClient client = buildClient(metalake); deleted = client.loadCatalog(catalog).asSchemas().dropSchema(schema, false); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTable.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTable.java index ee46d7f385e..be4e8466204 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTable.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTable.java @@ -77,20 +77,15 @@ public void handle() { NameIdentifier name = NameIdentifier.of(schema, table); deleted = client.loadCatalog(catalog).asTableCatalog().dropTable(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTag.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTag.java index 5b094fc605c..d3db384c094 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTag.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTag.java @@ -80,14 +80,11 @@ private void handleMultipleTags() { } } } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchTagException err) { - System.err.println(ErrorMessages.UNKNOWN_TAG); - return; + exitWithError(ErrorMessages.UNKNOWN_TAG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (!deleted.isEmpty()) { System.out.println("Tags " + String.join(",", deleted) + " deleted."); @@ -106,14 +103,11 @@ private void handleOnlyOneTag() { GravitinoClient client = buildClient(metalake); deleted = client.deleteTag(tags[0]); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchTagException err) { - System.err.println(ErrorMessages.UNKNOWN_TAG); - return; + exitWithError(ErrorMessages.UNKNOWN_TAG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTopic.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTopic.java index 1d1730652f4..5d6f440dba9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTopic.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteTopic.java @@ -77,17 +77,13 @@ public void handle() { GravitinoClient client = buildClient(metalake); deleted = client.loadCatalog(catalog).asTopicCatalog().dropTopic(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTopicException err) { - System.err.println(ErrorMessages.UNKNOWN_TOPIC); - return; + exitWithError(ErrorMessages.UNKNOWN_TOPIC); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteUser.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteUser.java index 6a748c9bbcd..3774c4501cb 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteUser.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteUser.java @@ -61,14 +61,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); deleted = client.removeUser(user); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchUserException err) { - System.err.println(ErrorMessages.UNKNOWN_USER); - return; + exitWithError(ErrorMessages.UNKNOWN_USER); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (deleted) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/FilesetDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/FilesetDetails.java index 8d7a5d2f392..3db387e39c7 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/FilesetDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/FilesetDetails.java @@ -70,20 +70,15 @@ public void handle() { GravitinoClient client = buildClient(metalake); result = client.loadCatalog(catalog).asFilesetCatalog().loadFileset(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchFilesetException err) { - System.err.println(ErrorMessages.UNKNOWN_FILESET); - return; + exitWithError(ErrorMessages.UNKNOWN_FILESET); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (result != null) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java index d21806ef299..4df87b5fa8e 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java @@ -53,14 +53,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); roles = client.getGroup(group).roles(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchUserException err) { - System.err.println(ErrorMessages.UNKNOWN_GROUP); - return; + exitWithError(ErrorMessages.UNKNOWN_GROUP); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", roles); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java index 83e97b7ac00..fa6c74c7afa 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java @@ -48,11 +48,9 @@ public void handle() { GravitinoClient client = buildClient(metalake); tags = client.listTags(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", tags); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogProperties.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogProperties.java index 638c6372a64..f94213eef42 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogProperties.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogProperties.java @@ -55,14 +55,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); gCatalog = client.loadCatalog(catalog); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } Map properties = gCatalog.properties(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java index eaff355e891..e6aaf811ec9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java @@ -51,9 +51,9 @@ public void handle() { catalogs = client.listCatalogsInfo(); output(catalogs); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); + exitWithError(exp.getMessage()); } } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListColumns.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListColumns.java index f289fbe475c..f3e8e0125cf 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListColumns.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListColumns.java @@ -66,8 +66,7 @@ public void handle() { ErrorMessages.UNKNOWN_TABLE + Joiner.on(".").join(metalake, catalog, schema, table)); return; } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } StringBuilder all = new StringBuilder(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListEntityTags.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListEntityTags.java index 9defeee8b93..a1c316fbdf2 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListEntityTags.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListEntityTags.java @@ -80,20 +80,15 @@ public void handle() { tags = gCatalog.supportsTags().listTags(); } } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", tags); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesetProperties.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesetProperties.java index 5175988345f..e090718abc3 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesetProperties.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesetProperties.java @@ -69,17 +69,13 @@ public void handle() { GravitinoClient client = buildClient(metalake); gFileset = client.loadCatalog(catalog).asFilesetCatalog().loadFileset(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } Map properties = gFileset.properties(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java index 428fe9bb1fb..34839f683c5 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java @@ -62,17 +62,13 @@ public void handle() { GravitinoClient client = buildClient(metalake); filesets = client.loadCatalog(catalog).asFilesetCatalog().listFilesets(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = Joiner.on(",").join(filesets); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java index d529b51479a..fd9009a755a 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java @@ -48,11 +48,9 @@ public void handle() { GravitinoClient client = buildClient(metalake); groups = client.listGroupNames(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", groups); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListIndexes.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListIndexes.java index 7178f8753ed..2d1a900baa8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListIndexes.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListIndexes.java @@ -54,14 +54,13 @@ public ListIndexes( /** Displays the details of a table's index. */ @Override public void handle() { - Index[] indexes; + Index[] indexes = {}; try { NameIdentifier name = NameIdentifier.of(schema, table); indexes = tableCatalog().loadTable(name).index(); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } StringBuilder all = new StringBuilder(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakeProperties.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakeProperties.java index d3349467478..b7d794d4455 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakeProperties.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakeProperties.java @@ -50,11 +50,9 @@ public void handle() { GravitinoAdminClient client = buildAdminClient(); gMetalake = client.loadMetalake(metalake); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } Map properties = gMetalake.properties(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java index de757156862..ee5ac81d646 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java @@ -45,7 +45,7 @@ public void handle() { metalakes = client.listMetalakes(); output(metalakes); } catch (Exception exp) { - System.err.println(exp.getMessage()); + exitWithError(exp.getMessage()); } } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java index cca26336e82..a7bb1cd20f7 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java @@ -48,11 +48,9 @@ public void handle() { GravitinoClient client = buildClient(metalake); roles = client.listRoleNames(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", roles); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java index 7a96e053dff..cf5fe487cc8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java @@ -53,14 +53,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); schemas = client.loadCatalog(catalog).asSchemas().listSchemas(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = Joiner.on(",").join(schemas); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchemaProperties.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchemaProperties.java index 210bda08744..76a99b8eb65 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchemaProperties.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchemaProperties.java @@ -59,17 +59,13 @@ public void handle() { GravitinoClient client = buildClient(metalake); gSchema = client.loadCatalog(catalog).asSchemas().loadSchema(schema); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } Map properties = gSchema.properties(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTableProperties.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTableProperties.java index a6d7d6a0ba2..61ebf5652dc 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTableProperties.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTableProperties.java @@ -70,20 +70,15 @@ public void handle() { GravitinoClient client = buildClient(metalake); gTable = client.loadCatalog(catalog).asTableCatalog().loadTable(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } Map properties = gTable.properties(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java index f5e27e608e2..e6afb9b51c0 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java @@ -53,8 +53,7 @@ public void handle() { try { tables = tableCatalog().listTables(name); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } List tableNames = new ArrayList<>(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTagProperties.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTagProperties.java index 14ea1d06603..5e191003ede 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTagProperties.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTagProperties.java @@ -54,14 +54,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); gTag = client.getTag(tag); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchTagException err) { - System.err.println(ErrorMessages.UNKNOWN_TAG); - return; + exitWithError(ErrorMessages.UNKNOWN_TAG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } Map properties = gTag.properties(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopicProperties.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopicProperties.java index 5063c61b99b..e308b1c6a15 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopicProperties.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopicProperties.java @@ -71,20 +71,15 @@ public void handle() { GravitinoClient client = buildClient(metalake); gTopic = client.loadCatalog(catalog).asTopicCatalog().loadTopic(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTopicException err) { - System.err.println(ErrorMessages.UNKNOWN_TOPIC); - return; + exitWithError(ErrorMessages.UNKNOWN_TOPIC); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } Map properties = gTopic.properties(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopics.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopics.java index 46416ff385b..af4cc217713 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopics.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopics.java @@ -61,11 +61,9 @@ public void handle() { GravitinoClient client = buildClient(metalake); topics = client.loadCatalog(catalog).asTopicCatalog().listTopics(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = Joiner.on(",").join(Arrays.stream(topics).map(topic -> topic.name()).iterator()); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListUsers.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListUsers.java index 465075a9786..a70176dcfcb 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListUsers.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListUsers.java @@ -48,11 +48,9 @@ public void handle() { GravitinoClient client = buildClient(metalake); users = client.listUserNames(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", users); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeAudit.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeAudit.java index 2e4b7c326bd..b966a1ae291 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeAudit.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeAudit.java @@ -43,15 +43,13 @@ public MetalakeAudit(String url, boolean ignoreVersions, String metalake) { /** Displays the audit information of a metalake. */ @Override public void handle() { - Audit audit; + Audit audit = null; try (GravitinoClient client = buildClient(metalake)) { audit = client.loadMetalake(metalake).auditInfo(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } displayAuditInfo(audit); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeDetails.java index 592317af4e6..ea503710d42 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeDetails.java @@ -49,9 +49,9 @@ public void handle() { Metalake metalakeEntity = client.loadMetalake(metalake); output(metalakeEntity); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); + exitWithError(exp.getMessage()); } } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/OwnerDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/OwnerDetails.java index 52cb4557237..8485a587560 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/OwnerDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/OwnerDetails.java @@ -75,14 +75,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); owner = client.getOwner(metadata); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchMetadataObjectException err) { - System.err.println(ErrorMessages.UNKNOWN_ENTITY); - return; + exitWithError(ErrorMessages.UNKNOWN_ENTITY); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (owner.isPresent()) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveAllTags.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveAllTags.java index 69f3bb7f0f1..a7aa3748a15 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveAllTags.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveAllTags.java @@ -100,20 +100,15 @@ public void handle() { entity = catalog; } } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (tags.length > 0) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveCatalogProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveCatalogProperty.java index 159e0137081..a460d91b2fe 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveCatalogProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveCatalogProperty.java @@ -57,14 +57,11 @@ public void handle() { CatalogChange change = CatalogChange.removeProperty(property); client.alterCatalog(catalog, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(property + " property removed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveFilesetProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveFilesetProperty.java index d32b195039b..00deebe265a 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveFilesetProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveFilesetProperty.java @@ -73,20 +73,15 @@ public void handle() { FilesetChange change = FilesetChange.removeProperty(property); client.loadCatalog(catalog).asFilesetCatalog().alterFileset(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchFilesetException err) { - System.err.println(ErrorMessages.UNKNOWN_FILESET); - return; + exitWithError(ErrorMessages.UNKNOWN_FILESET); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(property + " property removed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveMetalakeProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveMetalakeProperty.java index 894b0e53ed2..9642456f375 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveMetalakeProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveMetalakeProperty.java @@ -53,11 +53,9 @@ public void handle() { MetalakeChange change = MetalakeChange.removeProperty(property); client.alterMetalake(metalake, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(property + " property removed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromGroup.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromGroup.java index 8c219386e10..dd7ccc0e79d 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromGroup.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromGroup.java @@ -59,17 +59,13 @@ public void handle() { roles.add(role); client.revokeRolesFromGroup(roles, group); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchRoleException err) { - System.err.println(ErrorMessages.UNKNOWN_ROLE); - return; + exitWithError(ErrorMessages.UNKNOWN_ROLE); } catch (NoSuchUserException err) { - System.err.println(ErrorMessages.UNKNOWN_USER); - return; + exitWithError(ErrorMessages.UNKNOWN_USER); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(role + " removed from " + group); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromUser.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromUser.java index 0822fadc784..85a1edbeae4 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromUser.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromUser.java @@ -59,17 +59,13 @@ public void handle() { roles.add(role); client.revokeRolesFromUser(roles, user); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchRoleException err) { - System.err.println(ErrorMessages.UNKNOWN_ROLE); - return; + exitWithError(ErrorMessages.UNKNOWN_ROLE); } catch (NoSuchUserException err) { - System.err.println(ErrorMessages.UNKNOWN_USER); - return; + exitWithError(ErrorMessages.UNKNOWN_USER); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(role + " removed from " + user); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveSchemaProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveSchemaProperty.java index 12678d83aba..6fc41c01252 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveSchemaProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveSchemaProperty.java @@ -66,17 +66,13 @@ public void handle() { SchemaChange change = SchemaChange.removeProperty(property); client.loadCatalog(catalog).asSchemas().alterSchema(schema, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(property + " property removed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTableProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTableProperty.java index a805808859b..8b3cd2383fb 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTableProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTableProperty.java @@ -73,20 +73,15 @@ public void handle() { TableChange change = TableChange.removeProperty(property); client.loadCatalog(catalog).asTableCatalog().alterTable(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(property + " property removed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTagProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTagProperty.java index fc94f3bc1cf..a91395baf8f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTagProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTagProperty.java @@ -57,14 +57,11 @@ public void handle() { TagChange change = TagChange.removeProperty(property); client.alterTag(tag, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchTagException err) { - System.err.println(ErrorMessages.UNKNOWN_TAG); - return; + exitWithError(ErrorMessages.UNKNOWN_TAG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(property + " property removed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTopicProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTopicProperty.java index 4c21317592b..a43820933e8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTopicProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveTopicProperty.java @@ -74,20 +74,15 @@ public void handle() { TopicChange change = TopicChange.removeProperty(property); client.loadCatalog(catalog).asTopicCatalog().alterTopic(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTopicException err) { - System.err.println(ErrorMessages.UNKNOWN_TOPIC); - return; + exitWithError(ErrorMessages.UNKNOWN_TOPIC); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(property + " property removed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RoleDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RoleDetails.java index 2c1613eded7..57e314ac56e 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RoleDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RoleDetails.java @@ -55,14 +55,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); objects = client.getRole(role).securableObjects(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchUserException err) { - System.err.println(ErrorMessages.UNKNOWN_GROUP); - return; + exitWithError(ErrorMessages.UNKNOWN_GROUP); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } for (SecurableObject object : objects) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SchemaAudit.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SchemaAudit.java index b34ab36cfbf..ea891964da6 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SchemaAudit.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SchemaAudit.java @@ -53,22 +53,18 @@ public SchemaAudit( /** Displays the audit information of schema. */ @Override public void handle() { - Schema result; + Schema result = null; try (GravitinoClient client = buildClient(metalake)) { result = client.loadCatalog(catalog).asSchemas().loadSchema(this.schema); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (result != null) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SchemaDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SchemaDetails.java index ef7b5fe9a92..7369c0d1b41 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SchemaDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SchemaDetails.java @@ -59,17 +59,13 @@ public void handle() { GravitinoClient client = buildClient(metalake); result = client.loadCatalog(catalog).asSchemas().loadSchema(schema); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (result != null) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ServerVersion.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ServerVersion.java index 51488f0b6b2..218fa71bb88 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ServerVersion.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ServerVersion.java @@ -42,8 +42,7 @@ public void handle() { GravitinoAdminClient client = buildAdminClient(); version = client.serverVersion().version(); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println("Apache Gravitino " + version); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetCatalogProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetCatalogProperty.java index 141dae3d08b..21b1a6f1c9f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetCatalogProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetCatalogProperty.java @@ -65,14 +65,11 @@ public void handle() { CatalogChange change = CatalogChange.setProperty(property, value); client.alterCatalog(catalog, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(catalog + " property set."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetFilesetProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetFilesetProperty.java index 052af660fbe..2c179db104c 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetFilesetProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetFilesetProperty.java @@ -77,20 +77,15 @@ public void handle() { FilesetChange change = FilesetChange.setProperty(property, value); client.loadCatalog(catalog).asFilesetCatalog().alterFileset(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchFilesetException err) { - System.err.println(ErrorMessages.UNKNOWN_FILESET); - return; + exitWithError(ErrorMessages.UNKNOWN_FILESET); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(schema + " property set."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetMetalakeProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetMetalakeProperty.java index 8b2a1bb8f39..817beaec91e 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetMetalakeProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetMetalakeProperty.java @@ -56,11 +56,9 @@ public void handle() { MetalakeChange change = MetalakeChange.setProperty(property, value); client.alterMetalake(metalake, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(metalake + " property set."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetOwner.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetOwner.java index be9f1af5404..45de461043a 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetOwner.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetOwner.java @@ -90,14 +90,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); client.setOwner(metadata, owner, ownerType); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchMetadataObjectException err) { - System.err.println(ErrorMessages.UNKNOWN_ENTITY); - return; + exitWithError(ErrorMessages.UNKNOWN_ENTITY); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println("Set owner to " + owner); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetSchemaProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetSchemaProperty.java index ef71a3dc273..cc6151eaa2c 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetSchemaProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetSchemaProperty.java @@ -70,17 +70,13 @@ public void handle() { SchemaChange change = SchemaChange.setProperty(property, value); client.loadCatalog(catalog).asSchemas().alterSchema(schema, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(schema + " property set."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTableProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTableProperty.java index 03fd787da2b..0209d218250 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTableProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTableProperty.java @@ -77,20 +77,15 @@ public void handle() { TableChange change = TableChange.setProperty(property, value); client.loadCatalog(catalog).asTableCatalog().alterTable(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(table + " property set."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTagProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTagProperty.java index 5b0a73d3b62..b5b46b59a71 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTagProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTagProperty.java @@ -65,14 +65,11 @@ public void handle() { TagChange change = TagChange.setProperty(property, value); client.alterTag(tag, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchTagException err) { - System.err.println(ErrorMessages.UNKNOWN_TAG); - return; + exitWithError(ErrorMessages.UNKNOWN_TAG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(tag + " property set."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTopicProperty.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTopicProperty.java index 55ed1c05c79..941c0b0321e 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTopicProperty.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/SetTopicProperty.java @@ -79,20 +79,15 @@ public void handle() { client.loadCatalog(catalog).asTopicCatalog().alterTopic(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTopicException err) { - System.err.println(ErrorMessages.UNKNOWN_TOPIC); - return; + exitWithError(ErrorMessages.UNKNOWN_TOPIC); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(property + " property set."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableAudit.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableAudit.java index 8051daf3450..0a89076f657 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableAudit.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableAudit.java @@ -53,14 +53,13 @@ public TableAudit( /** Displays the audit information of a table. */ @Override public void handle() { - Table gTable; + Table gTable = null; try { NameIdentifier name = NameIdentifier.of(schema, table); gTable = tableCatalog().loadTable(name); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } displayAuditInfo(gTable.auditInfo()); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableCommand.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableCommand.java index f0e89d71ff5..8ade3c11a78 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableCommand.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableCommand.java @@ -61,15 +61,15 @@ public TableCatalog tableCatalog() { GravitinoClient client = buildClient(metalake); return client.loadMetalake(metalake).loadCatalog(catalog).asTableCatalog(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); + exitWithError(exp.getMessage()); } return null; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableDetails.java index 3f8dcfd2b51..0f38218f7c8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableDetails.java @@ -59,8 +59,7 @@ public void handle() { NameIdentifier name = NameIdentifier.of(schema, table); gTable = tableCatalog().loadTable(name); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(gTable.name() + "," + gTable.comment()); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableDistribution.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableDistribution.java index 7fac9b861d8..72a0f3ef3fb 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableDistribution.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableDistribution.java @@ -53,14 +53,13 @@ public TableDistribution( /** Displays the strategy and bucket number of distirbution. */ @Override public void handle() { - Distribution distribution; + Distribution distribution = null; try { NameIdentifier name = NameIdentifier.of(schema, table); distribution = tableCatalog().loadTable(name).distribution(); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(distribution.strategy() + "," + distribution.number()); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TablePartition.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TablePartition.java index 3726d690dbf..bbf86303d5b 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TablePartition.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TablePartition.java @@ -54,13 +54,12 @@ public TablePartition( /** Displays the name and properties of partition. */ @Override public void handle() { - Transform transforms[]; + Transform transforms[] = {}; try { NameIdentifier name = NameIdentifier.of(schema, table); transforms = tableCatalog().loadTable(name).partitioning(); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } for (Transform transform : transforms) { Partition[] partitions = transform.assignments(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableSortOrder.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableSortOrder.java index 0da8f6501b0..54fc1cca273 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableSortOrder.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TableSortOrder.java @@ -53,14 +53,13 @@ public TableSortOrder( /** Displays the expression, direction and nullOrdering number of sort order. */ @Override public void handle() { - SortOrder[] sortOrders; + SortOrder[] sortOrders = {}; try { NameIdentifier name = NameIdentifier.of(schema, table); sortOrders = tableCatalog().loadTable(name).sortOrder(); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } for (SortOrder sortOrder : sortOrders) { System.out.printf( diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagDetails.java index 711d330493a..871a7e70742 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagDetails.java @@ -53,14 +53,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); result = client.getTag(tag); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } if (result != null) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java index b6f7c3210e0..55bb4b7f436 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java @@ -87,20 +87,15 @@ public void handle() { entity = catalog; } } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", tagsToAdd); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TopicDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TopicDetails.java index 0e73bb8450d..0ab31bd8b36 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TopicDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TopicDetails.java @@ -70,20 +70,15 @@ public void handle() { GravitinoClient client = buildClient(metalake); gTopic = client.loadCatalog(catalog).asTopicCatalog().loadTopic(name); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTopicException err) { - System.err.println(ErrorMessages.UNKNOWN_TOPIC); - return; + exitWithError(ErrorMessages.UNKNOWN_TOPIC); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(gTopic.name() + "," + gTopic.comment()); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java index 3b9771bc8fe..36b806056cc 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java @@ -87,20 +87,15 @@ public void handle() { entity = catalog; } } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", removeTags); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogComment.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogComment.java index eae5e0d1128..ed12dbc7caa 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogComment.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogComment.java @@ -57,14 +57,11 @@ public void handle() { CatalogChange change = CatalogChange.updateComment(comment); client.alterCatalog(catalog, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(catalog + " comment changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogName.java index e4339e2460e..399d600fcef 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogName.java @@ -57,14 +57,11 @@ public void handle() { CatalogChange change = CatalogChange.rename(name); client.alterCatalog(catalog, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(catalog + " name changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnAutoIncrement.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnAutoIncrement.java index f848e6a680d..99b27b200e8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnAutoIncrement.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnAutoIncrement.java @@ -81,23 +81,17 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(columnName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchColumnException err) { - System.err.println(ErrorMessages.UNKNOWN_COLUMN); - return; + exitWithError(ErrorMessages.UNKNOWN_COLUMN); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " auto increment changed to " + autoincrement + "."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnComment.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnComment.java index 09d73ccb984..2c7f05a8fd9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnComment.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnComment.java @@ -81,23 +81,17 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(columnName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchColumnException err) { - System.err.println(ErrorMessages.UNKNOWN_COLUMN); - return; + exitWithError(ErrorMessages.UNKNOWN_COLUMN); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " comment changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnDatatype.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnDatatype.java index 6a40be9537a..6eac9da7ef9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnDatatype.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnDatatype.java @@ -84,23 +84,17 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(columnName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchColumnException err) { - System.err.println(ErrorMessages.UNKNOWN_COLUMN); - return; + exitWithError(ErrorMessages.UNKNOWN_COLUMN); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " datatype changed to " + datatype + "."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnDefault.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnDefault.java index de5a00c1f38..7c7c2d3b402 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnDefault.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnDefault.java @@ -88,23 +88,17 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(columnName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchColumnException err) { - System.err.println(ErrorMessages.UNKNOWN_COLUMN); - return; + exitWithError(ErrorMessages.UNKNOWN_COLUMN); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " default changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnName.java index f03d0ce8dab..124223dbd29 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnName.java @@ -81,23 +81,17 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(columnName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchColumnException err) { - System.err.println(ErrorMessages.UNKNOWN_COLUMN); - return; + exitWithError(ErrorMessages.UNKNOWN_COLUMN); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " name changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnNullability.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnNullability.java index 88f3634e0d2..868b426868f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnNullability.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnNullability.java @@ -81,23 +81,17 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(columnName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchColumnException err) { - System.err.println(ErrorMessages.UNKNOWN_COLUMN); - return; + exitWithError(ErrorMessages.UNKNOWN_COLUMN); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " nullability changed to " + nullability + "."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnPosition.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnPosition.java index 460fe6ed159..4bbe809d448 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnPosition.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateColumnPosition.java @@ -84,23 +84,17 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(columnName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (NoSuchColumnException err) { - System.err.println(ErrorMessages.UNKNOWN_COLUMN); - return; + exitWithError(ErrorMessages.UNKNOWN_COLUMN); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(column + " position changed to " + position + "."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetComment.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetComment.java index 36c5ed2285b..886c3dc3cdc 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetComment.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetComment.java @@ -74,20 +74,15 @@ public void handle() { client.loadCatalog(catalog).asFilesetCatalog().alterFileset(filesetName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchFilesetException err) { - System.err.println(ErrorMessages.UNKNOWN_FILESET); - return; + exitWithError(ErrorMessages.UNKNOWN_FILESET); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(fileset + " comment changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetName.java index 11dd1c8c863..6d4ca8e0f27 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetName.java @@ -73,20 +73,15 @@ public void handle() { FilesetChange change = FilesetChange.rename(name); client.loadCatalog(catalog).asFilesetCatalog().alterFileset(filesetName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchFilesetException err) { - System.err.println(ErrorMessages.UNKNOWN_FILESET); - return; + exitWithError(ErrorMessages.UNKNOWN_FILESET); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(fileset + " name changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateMetalakeComment.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateMetalakeComment.java index ae4412587e8..9ca63084e75 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateMetalakeComment.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateMetalakeComment.java @@ -53,11 +53,9 @@ public void handle() { MetalakeChange change = MetalakeChange.updateComment(comment); client.alterMetalake(metalake, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(metalake + " comment changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateMetalakeName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateMetalakeName.java index acf5470af8d..275ba3165df 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateMetalakeName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateMetalakeName.java @@ -62,11 +62,9 @@ public void handle() { MetalakeChange change = MetalakeChange.rename(name); client.alterMetalake(metalake, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(metalake + " name changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableComment.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableComment.java index 3edd739406d..c71795a9ec4 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableComment.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableComment.java @@ -74,20 +74,15 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(tableName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(table + " comment changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableName.java index 2346ee4ab3c..773d366fb3b 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableName.java @@ -74,20 +74,15 @@ public void handle() { client.loadCatalog(catalog).asTableCatalog().alterTable(tableName, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { - System.err.println(ErrorMessages.UNKNOWN_TABLE); - return; + exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(table + " name changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagComment.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagComment.java index 17ad6f7e3cb..2994d78f302 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagComment.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagComment.java @@ -57,14 +57,11 @@ public void handle() { TagChange change = TagChange.updateComment(comment); client.alterTag(tag, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchTagException err) { - System.err.println(ErrorMessages.UNKNOWN_TAG); - return; + exitWithError(ErrorMessages.UNKNOWN_TAG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(tag + " comment changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagName.java index 890cd26cfc7..96fb9d15714 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagName.java @@ -57,14 +57,11 @@ public void handle() { TagChange change = TagChange.rename(name); client.alterTag(tag, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchTagException err) { - System.err.println(ErrorMessages.UNKNOWN_TAG); - return; + exitWithError(ErrorMessages.UNKNOWN_TAG); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(tag + " name changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTopicComment.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTopicComment.java index d57efcaf90e..1f81ad7fab3 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTopicComment.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTopicComment.java @@ -74,20 +74,15 @@ public void handle() { TopicChange change = TopicChange.updateComment(comment); client.loadCatalog(catalog).asTopicCatalog().alterTopic(name, change); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { - System.err.println(ErrorMessages.UNKNOWN_CATALOG); - return; + exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - System.err.println(ErrorMessages.UNKNOWN_SCHEMA); - return; + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTopicException err) { - System.err.println(ErrorMessages.UNKNOWN_TOPIC); - return; + exitWithError(ErrorMessages.UNKNOWN_TOPIC); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } System.out.println(topic + " comment changed."); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java index 43ebb65e8d4..1d59c83e529 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java @@ -53,14 +53,11 @@ public void handle() { GravitinoClient client = buildClient(metalake); roles = client.getUser(user).roles(); } catch (NoSuchMetalakeException err) { - System.err.println(ErrorMessages.UNKNOWN_METALAKE); - return; + exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchUserException err) { - System.err.println(ErrorMessages.UNKNOWN_USER); - return; + exitWithError(ErrorMessages.UNKNOWN_USER); } catch (Exception exp) { - System.err.println(exp.getMessage()); - return; + exitWithError(exp.getMessage()); } String all = String.join(",", roles); diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java index d4681d8c235..e26759e2d4c 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java @@ -366,7 +366,6 @@ void testUpdateColumnDefault() { when(mockCommandLine.getOptionValue(GravitinoOptions.DEFAULT)).thenReturn("Fred Smith"); when(mockCommandLine.hasOption(GravitinoOptions.DATATYPE)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.DATATYPE)).thenReturn("varchar(100)"); - when(mockCommandLine.hasOption(GravitinoOptions.NULL)).thenReturn(false); when(mockCommandLine.hasOption(GravitinoOptions.AUTO)).thenReturn(false); diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java index 00fe52e9fe9..3f1c4a4cb1e 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java @@ -215,15 +215,16 @@ void testRemoveRolesFromGroupCommand() { .when(commandLine) .newRemoveRoleFromGroup( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "groupA", "admin"); - commandLine.handleCommandLine(); - verify(mockRemoveFirstRole).handle(); // Verify second role doReturn(mockRemoveSecondRole) .when(commandLine) .newRemoveRoleFromGroup( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "groupA", "role1"); + commandLine.handleCommandLine(); + + verify(mockRemoveFirstRole).handle(); verify(mockRemoveSecondRole).handle(); } @@ -247,15 +248,16 @@ void testAddRolesToGroupCommand() { .when(commandLine) .newAddRoleToGroup( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "groupA", "admin"); - commandLine.handleCommandLine(); - verify(mockAddFirstRole).handle(); // Verify second role doReturn(mockAddSecondRole) .when(commandLine) .newAddRoleToGroup( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "groupA", "role1"); + commandLine.handleCommandLine(); + verify(mockAddSecondRole).handle(); + verify(mockAddFirstRole).handle(); } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java index 302e8af993b..93de0a6bc9d 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; @@ -125,7 +126,12 @@ public void withHelpOption() throws ParseException, UnsupportedEncodingException public void parseError() throws UnsupportedEncodingException { String[] args = {"--invalidOption"}; - Main.main(args); + Main.useExit = false; + assertThrows( + RuntimeException.class, + () -> { + Main.main(args); + }); assertTrue(errContent.toString().contains("Error parsing command line")); // Expect error assertTrue(outContent.toString().contains("usage:")); // Expect help output diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java index 21c5743643d..e8a1864b9ff 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java @@ -216,16 +216,17 @@ void testRemoveRolesFromUserCommand() { .when(commandLine) .newRemoveRoleFromUser( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "user", "admin"); - commandLine.handleCommandLine(); - verify(mockRemoveFirstRole).handle(); // Verify second role doReturn(mockRemoveSecondRole) .when(commandLine) .newRemoveRoleFromUser( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "user", "role1"); + commandLine.handleCommandLine(); + verify(mockRemoveSecondRole).handle(); + verify(mockRemoveFirstRole).handle(); } @Test @@ -249,15 +250,16 @@ void testAddRolesToUserCommand() { .when(commandLine) .newAddRoleToUser( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "user", "admin"); - commandLine.handleCommandLine(); - verify(mockAddFirstRole).handle(); // Verify second role doReturn(mockAddSecondRole) .when(commandLine) .newAddRoleToUser( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "user", "role1"); + commandLine.handleCommandLine(); + + verify(mockAddFirstRole).handle(); verify(mockAddSecondRole).handle(); } } From f850e4affd60fa9c687a3abf3052603c83219028 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 19 Dec 2024 22:21:38 +0800 Subject: [PATCH 07/47] [#5892] fix(auth): Fix to grant privilege for the metalake (#5919) ### What changes were proposed in this pull request? Fix to grant privilege for the metalake ### Why are the changes needed? Fix: #5892 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add a UT. --- .../integration/test/RangerBaseE2EIT.java | 33 +++++++++++++++++++ .../authorization/AuthorizationUtils.java | 13 ++++++-- ...estAccessControlManagerForPermissions.java | 6 ++-- .../authorization/TestOwnerManager.java | 8 +++-- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java index 95dc4f93636..de5641ffc7d 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java @@ -984,4 +984,37 @@ void testDenyPrivileges() throws InterruptedException { catalog.asSchemas().dropSchema(schemaName, false); metalake.deleteRole(roleName); } + + // ISSUE-5892 Fix to grant privilege for the metalake + @Test + void testGrantPrivilegesForMetalake() throws InterruptedException { + // Choose a catalog + useCatalog(); + + // Create a schema + String roleName = currentFunName(); + metalake.createRole(roleName, Collections.emptyMap(), Collections.emptyList()); + + // Grant a create schema privilege + metalake.grantPrivilegesToRole( + roleName, + MetadataObjects.of(null, metalakeName, MetadataObject.Type.METALAKE), + Lists.newArrayList(Privileges.CreateSchema.allow())); + + // Fail to create a schema + Assertions.assertThrows( + AccessControlException.class, () -> sparkSession.sql(SQL_CREATE_SCHEMA)); + + // Granted this role to the spark execution user `HADOOP_USER_NAME` + String userName1 = System.getenv(HADOOP_USER_NAME); + metalake.grantRolesToUser(Lists.newArrayList(roleName), userName1); + + waitForUpdatingPolicies(); + + Assertions.assertDoesNotThrow(() -> sparkSession.sql(SQL_CREATE_SCHEMA)); + + // Clean up + catalog.asSchemas().dropSchema(schemaName, false); + metalake.deleteRole(roleName); + } } diff --git a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java index ca5866558b4..793b478eb6d 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java @@ -173,9 +173,11 @@ public static void callAuthorizationPluginForMetadataObject( String metalake, MetadataObject metadataObject, Consumer consumer) { CatalogManager catalogManager = GravitinoEnv.getInstance().catalogManager(); if (needApplyAuthorizationPluginAllCatalogs(metadataObject.type())) { - Catalog[] catalogs = catalogManager.listCatalogsInfo(Namespace.of(metalake)); - for (Catalog catalog : catalogs) { - callAuthorizationPluginImpl(consumer, catalog); + NameIdentifier[] catalogs = catalogManager.listCatalogs(Namespace.of(metalake)); + // ListCatalogsInfo return `CatalogInfo` instead of `BaseCatalog`, we need `BaseCatalog` to + // call authorization plugin method. + for (NameIdentifier catalog : catalogs) { + callAuthorizationPluginImpl(consumer, catalogManager.loadCatalog(catalog)); } } else if (needApplyAuthorization(metadataObject.type())) { NameIdentifier catalogIdent = @@ -269,6 +271,11 @@ private static void callAuthorizationPluginImpl( if (baseCatalog.getAuthorizationPlugin() != null) { consumer.accept(baseCatalog.getAuthorizationPlugin()); } + } else { + throw new IllegalArgumentException( + String.format( + "Catalog %s is not a BaseCatalog, we don't support authorization plugin for it", + catalog.type())); } } diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java index d0c2b1b2087..30084a32e2d 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -28,7 +28,6 @@ import java.time.Instant; import java.util.List; import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.gravitino.Catalog; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; import org.apache.gravitino.Entity; @@ -36,6 +35,7 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.catalog.CatalogManager; import org.apache.gravitino.connector.BaseCatalog; @@ -172,8 +172,8 @@ public static void setUp() throws Exception { FieldUtils.writeField(GravitinoEnv.getInstance(), "catalogManager", catalogManager, true); BaseCatalog catalog = Mockito.mock(BaseCatalog.class); Mockito.when(catalogManager.loadCatalog(any())).thenReturn(catalog); - Mockito.when(catalogManager.listCatalogsInfo(Mockito.any())) - .thenReturn(new Catalog[] {catalog}); + Mockito.when(catalogManager.listCatalogs(Mockito.any())) + .thenReturn(new NameIdentifier[] {NameIdentifier.of("metalake", "catalog")}); authorizationPlugin = Mockito.mock(AuthorizationPlugin.class); Mockito.when(catalog.getAuthorizationPlugin()).thenReturn(authorizationPlugin); } diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java b/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java index 83a562f640d..d4def869f73 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java @@ -31,6 +31,7 @@ import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.VERSION_RETENTION_COUNT; +import static org.mockito.ArgumentMatchers.any; import com.google.common.collect.Lists; import java.io.File; @@ -40,13 +41,13 @@ import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.gravitino.Catalog; import org.apache.gravitino.Config; import org.apache.gravitino.EntityStore; import org.apache.gravitino.EntityStoreFactory; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.catalog.CatalogManager; import org.apache.gravitino.connector.BaseCatalog; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; @@ -145,8 +146,9 @@ public static void setUp() throws IOException, IllegalAccessException { ownerManager = new OwnerManager(entityStore); BaseCatalog catalog = Mockito.mock(BaseCatalog.class); - Mockito.when(catalogManager.listCatalogsInfo(Mockito.any())) - .thenReturn(new Catalog[] {catalog}); + Mockito.when(catalogManager.loadCatalog(any())).thenReturn(catalog); + Mockito.when(catalogManager.listCatalogs(Mockito.any())) + .thenReturn(new NameIdentifier[] {NameIdentifier.of("metalake", "catalog")}); Mockito.when(catalog.getAuthorizationPlugin()).thenReturn(authorizationPlugin); } From 1d16bda198fca52d62cea9415a76c4f7074043ec Mon Sep 17 00:00:00 2001 From: Xun Date: Fri, 20 Dec 2024 11:43:52 +0800 Subject: [PATCH 08/47] [#5916] improve(auth): Remove AuthorizationPlugin single instance implement (#5918) ### What changes were proposed in this pull request? 1. Remove AuthorizationPlugin single instance implement in the `BaseAuthorizaiton.java` 2. Updatge ITs codes. ### Why are the changes needed? Fix: #5916 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? CI Passed. --- .../ranger/RangerAuthorization.java | 6 +- .../ranger/RangerAuthorizationHDFSPlugin.java | 16 +-- .../RangerAuthorizationHadoopSQLPlugin.java | 15 +-- .../ranger/RangerAuthorizationPlugin.java | 24 ++-- .../test/RangerAuthorizationHDFSPluginIT.java | 2 +- .../test/RangerAuthorizationPluginIT.java | 2 +- .../integration/test/RangerBaseE2EIT.java | 12 +- .../integration/test/RangerFilesetIT.java | 2 +- .../integration/test/RangerHiveE2EIT.java | 12 +- .../ranger/integration/test/RangerHiveIT.java | 2 +- .../ranger/integration/test/RangerITEnv.java | 18 +-- .../integration/test/RangerIcebergE2EIT.java | 5 +- .../integration/test/RangerPaimonE2EIT.java | 5 +- .../gravitino/connector/BaseCatalog.java | 106 ++++++++++-------- .../authorization/BaseAuthorization.java | 22 +--- .../mysql/TestMySQLAuthorization.java | 2 +- .../ranger/TestRangerAuthorization.java | 2 +- 17 files changed, 115 insertions(+), 138 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java index cd27d9f12a2..6aae714a359 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java @@ -33,7 +33,7 @@ public String shortName() { } @Override - protected AuthorizationPlugin newPlugin( + public AuthorizationPlugin newPlugin( String metalake, String catalogProvider, Map properties) { Preconditions.checkArgument( properties.containsKey(RANGER_SERVICE_TYPE), @@ -41,9 +41,9 @@ protected AuthorizationPlugin newPlugin( String serviceType = properties.get(RANGER_SERVICE_TYPE).toUpperCase(); switch (serviceType) { case "HADOOPSQL": - return RangerAuthorizationHadoopSQLPlugin.getInstance(metalake, properties); + return new RangerAuthorizationHadoopSQLPlugin(metalake, properties); case "HDFS": - return RangerAuthorizationHDFSPlugin.getInstance(metalake, properties); + return new RangerAuthorizationHDFSPlugin(metalake, properties); default: throw new IllegalArgumentException("Unsupported service type: " + serviceType); } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 16ce5bba4cb..9afa77880e9 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -52,24 +52,10 @@ public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { private static final Pattern pattern = Pattern.compile("^hdfs://[^/]*"); - private static volatile RangerAuthorizationHDFSPlugin instance = null; - - private RangerAuthorizationHDFSPlugin(String metalake, Map config) { + public RangerAuthorizationHDFSPlugin(String metalake, Map config) { super(metalake, config); } - public static synchronized RangerAuthorizationHDFSPlugin getInstance( - String metalake, Map config) { - if (instance == null) { - synchronized (RangerAuthorizationHadoopSQLPlugin.class) { - if (instance == null) { - instance = new RangerAuthorizationHDFSPlugin(metalake, config); - } - } - } - return instance; - } - @Override public Map> privilegesMappingRule() { return ImmutableMap.of( diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java index 0da5c105a4b..b8e078d086e 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java @@ -48,24 +48,11 @@ public class RangerAuthorizationHadoopSQLPlugin extends RangerAuthorizationPlugin { private static final Logger LOG = LoggerFactory.getLogger(RangerAuthorizationHadoopSQLPlugin.class); - private static volatile RangerAuthorizationHadoopSQLPlugin instance = null; - private RangerAuthorizationHadoopSQLPlugin(String metalake, Map config) { + public RangerAuthorizationHadoopSQLPlugin(String metalake, Map config) { super(metalake, config); } - public static synchronized RangerAuthorizationHadoopSQLPlugin getInstance( - String metalake, Map config) { - if (instance == null) { - synchronized (RangerAuthorizationHadoopSQLPlugin.class) { - if (instance == null) { - instance = new RangerAuthorizationHadoopSQLPlugin(metalake, config); - } - } - } - return instance; - } - @Override /** Set the default mapping Gravitino privilege name to the Ranger rule */ public Map> privilegesMappingRule() { diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 9c30ee11906..7a91ad54bf0 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -226,7 +226,7 @@ public Boolean onRoleUpdated(Role role, RoleChange... changes) SecurableObject securableObject = ((RoleChange.AddSecurableObject) change).getSecurableObject(); if (!validAuthorizationOperation(Arrays.asList(securableObject))) { - return false; + return Boolean.FALSE; } List AuthorizationSecurableObjects = @@ -243,7 +243,7 @@ public Boolean onRoleUpdated(Role role, RoleChange... changes) SecurableObject securableObject = ((RoleChange.RemoveSecurableObject) change).getSecurableObject(); if (!validAuthorizationOperation(Arrays.asList(securableObject))) { - return false; + return Boolean.FALSE; } List AuthorizationSecurableObjects = @@ -260,12 +260,12 @@ public Boolean onRoleUpdated(Role role, RoleChange... changes) SecurableObject oldSecurableObject = ((RoleChange.UpdateSecurableObject) change).getSecurableObject(); if (!validAuthorizationOperation(Arrays.asList(oldSecurableObject))) { - return false; + return Boolean.FALSE; } SecurableObject newSecurableObject = ((RoleChange.UpdateSecurableObject) change).getNewSecurableObject(); if (!validAuthorizationOperation(Arrays.asList(newSecurableObject))) { - return false; + return Boolean.FALSE; } Preconditions.checkArgument( @@ -394,8 +394,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n onGroupAdded(groupEntity); } - List AuthorizationSecurableObjects = - translateOwner(metadataObject); + List rangerSecurableObjects = translateOwner(metadataObject); String ownerRoleName; switch (metadataObject.type()) { case METALAKE: @@ -426,14 +425,13 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n LOG.warn("Grant owner role: {} failed!", ownerRoleName, e); } - AuthorizationSecurableObjects.stream() + rangerSecurableObjects.stream() .forEach( - AuthorizationSecurableObject -> { - RangerPolicy policy = - rangerHelper.findManagedPolicy(AuthorizationSecurableObject); + rangerSecurableObject -> { + RangerPolicy policy = rangerHelper.findManagedPolicy(rangerSecurableObject); try { if (policy == null) { - policy = addOwnerRoleToNewPolicy(AuthorizationSecurableObject, ownerRoleName); + policy = addOwnerRoleToNewPolicy(rangerSecurableObject, ownerRoleName); rangerClient.createPolicy(policy); } else { rangerHelper.updatePolicyOwnerRole(policy, ownerRoleName); @@ -449,7 +447,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n case TABLE: case FILESET: // The schema and table use user/group to manage the owner - AuthorizationSecurableObjects.stream() + rangerSecurableObjects.stream() .forEach( AuthorizationSecurableObject -> { RangerPolicy policy = @@ -483,7 +481,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n * 2. Create a role in the Ranger if the role does not exist.
* 3. Add this user to the role.
* - * @param roles The roles to grant to the group. + * @param roles The roles to grant to the user. * @param user The user to grant the roles. */ @Override diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index e1eacba1587..4062263222b 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -42,7 +42,7 @@ public class RangerAuthorizationHDFSPluginIT { @BeforeAll public static void setup() { - RangerITEnv.init(true); + RangerITEnv.init(RangerITEnv.currentFunName(), true); rangerAuthPlugin = RangerITEnv.rangerAuthHDFSPlugin; } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java index 74ddf078491..881d8f0ab44 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java @@ -45,7 +45,7 @@ public class RangerAuthorizationPluginIT { @BeforeAll public static void setup() { - RangerITEnv.init(true); + RangerITEnv.init(RangerITEnv.currentFunName(), true); rangerAuthPlugin = RangerITEnv.rangerAuthHivePlugin; } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java index de5641ffc7d..c7c9ec02f22 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java @@ -169,12 +169,18 @@ protected void createMetalake() { metalake = loadMetalake; } - protected static void waitForUpdatingPolicies() throws InterruptedException { + public abstract void createCatalog(); + + protected static void waitForUpdatingPolicies() { // After Ranger authorization, Must wait a period of time for the Ranger Spark plugin to update // the policy Sleep time must be greater than the policy update interval // (ranger.plugin.spark.policy.pollIntervalMs) in the // `resources/ranger-spark-security.xml.template` - Thread.sleep(1000L); + try { + Thread.sleep(1000L); + } catch (InterruptedException e) { + LOG.error("Failed to sleep", e); + } } protected abstract void checkTableAllPrivilegesExceptForCreating(); @@ -198,7 +204,7 @@ protected static void waitForUpdatingPolicies() throws InterruptedException { protected abstract void testAlterTable(); @Test - void testCreateSchema() throws InterruptedException { + protected void testCreateSchema() throws InterruptedException { // Choose a catalog useCatalog(); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java index 56f09781587..d8024afcc11 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java @@ -95,7 +95,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(false); + RangerITEnv.init(metalakeName, false); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index baec9434c79..363f8f0b3a1 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -31,6 +31,7 @@ import org.apache.gravitino.auth.AuthenticatorType; import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; import org.apache.gravitino.catalog.hive.HiveConstants; +import org.apache.gravitino.exceptions.UserAlreadyExistsException; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.RangerContainer; import org.apache.gravitino.integration.test.util.GravitinoITUtils; @@ -63,7 +64,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(true); + RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = @@ -102,7 +103,11 @@ public void startIntegrationTest() throws Exception { createCatalog(); RangerITEnv.cleanup(); - metalake.addUser(System.getenv(HADOOP_USER_NAME)); + try { + metalake.addUser(System.getenv(HADOOP_USER_NAME)); + } catch (UserAlreadyExistsException e) { + LOG.error("Failed to add user: {}", System.getenv(HADOOP_USER_NAME), e); + } } @AfterAll @@ -166,7 +171,8 @@ protected void testAlterTable() { sparkSession.sql(SQL_ALTER_TABLE); } - private static void createCatalog() { + @Override + public void createCatalog() { Map properties = ImmutableMap.of( HiveConstants.METASTORE_URIS, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java index 9c45a21099e..9545f243dd3 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -80,7 +80,7 @@ public class RangerHiveIT { @BeforeAll public static void setup() { - RangerITEnv.init(true); + RangerITEnv.init(RangerITEnv.currentFunName(), true); rangerAuthHivePlugin = RangerITEnv.rangerAuthHivePlugin; rangerHelper = RangerITEnv.rangerHelper; diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index b3be410ea03..2efc1e9dd60 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -59,12 +59,12 @@ public class RangerITEnv { private static final Logger LOG = LoggerFactory.getLogger(RangerITEnv.class); protected static final String RANGER_TRINO_REPO_NAME = "trinoDev"; private static final String RANGER_TRINO_TYPE = "trino"; - protected static final String RANGER_HIVE_REPO_NAME = "hiveDev"; + public static final String RANGER_HIVE_REPO_NAME = "hiveDev"; private static final String RANGER_HIVE_TYPE = "hive"; - protected static final String RANGER_HDFS_REPO_NAME = "hdfsDev"; + public static final String RANGER_HDFS_REPO_NAME = "hdfsDev"; private static final String RANGER_HDFS_TYPE = "hdfs"; protected static RangerClient rangerClient; - protected static final String HADOOP_USER_NAME = "gravitino"; + public static final String HADOOP_USER_NAME = "gravitino"; private static volatile boolean initRangerService = false; private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); @@ -90,13 +90,13 @@ public class RangerITEnv { protected static RangerHelper rangerHDFSHelper; - public static void init(boolean allowAnyoneAccessHDFS) { + public static void init(String metalakeName, boolean allowAnyoneAccessHDFS) { containerSuite.startRangerContainer(); rangerClient = containerSuite.getRangerContainer().rangerClient; rangerAuthHivePlugin = - RangerAuthorizationHadoopSQLPlugin.getInstance( - "metalake", + new RangerAuthorizationHadoopSQLPlugin( + metalakeName, ImmutableMap.of( RangerAuthorizationProperties.RANGER_ADMIN_URL, String.format( @@ -116,8 +116,8 @@ public static void init(boolean allowAnyoneAccessHDFS) { RangerAuthorizationHDFSPlugin spyRangerAuthorizationHDFSPlugin = Mockito.spy( - RangerAuthorizationHDFSPlugin.getInstance( - "metalake", + new RangerAuthorizationHDFSPlugin( + metalakeName, ImmutableMap.of( RangerAuthorizationProperties.RANGER_ADMIN_URL, String.format( @@ -175,7 +175,7 @@ public static void cleanup() { } } - static void startHiveRangerContainer() { + public static void startHiveRangerContainer() { containerSuite.startHiveRangerContainer( new HashMap<>( ImmutableMap.of( diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java index d8bd70c6470..8f6f769504a 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java @@ -67,7 +67,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(true); + RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = @@ -163,7 +163,8 @@ protected void testAlterTable() { sparkSession.sql(SQL_ALTER_TABLE_BACK); } - private static void createCatalog() { + @Override + public void createCatalog() { Map properties = new HashMap<>(); properties.put(IcebergConstants.URI, HIVE_METASTORE_URIS); properties.put(IcebergConstants.CATALOG_BACKEND, "hive"); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java index 79d1eb1875d..2773610048e 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java @@ -66,7 +66,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(true); + RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = @@ -179,7 +179,8 @@ protected void testAlterTable() { sparkSession.sql(SQL_ALTER_TABLE_BACK); } - private static void createCatalog() { + @Override + public void createCatalog() { Map properties = ImmutableMap.of( "uri", diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java index dbc9c085968..88fd47ab998 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java @@ -66,7 +66,7 @@ public abstract class BaseCatalog public static final String CATALOG_OPERATION_IMPL = "ops-impl"; // Underlying access control system plugin for this catalog. - private volatile BaseAuthorization authorization; + private volatile AuthorizationPlugin authorizationPlugin; private CatalogEntity entity; @@ -187,54 +187,64 @@ public CatalogOperations ops() { } public AuthorizationPlugin getAuthorizationPlugin() { - if (authorization == null) { - return null; + if (authorizationPlugin == null) { + synchronized (this) { + if (authorizationPlugin == null) { + return null; + } + } } - return authorization.plugin(entity.namespace().level(0), provider(), this.conf); + return authorizationPlugin; } public void initAuthorizationPluginInstance(IsolatedClassLoader classLoader) { - if (authorization != null) { - return; - } - - String authorizationProvider = - (String) catalogPropertiesMetadata().getOrDefault(conf, AUTHORIZATION_PROVIDER); - if (authorizationProvider == null) { - LOG.info("Authorization provider is not set!"); - return; - } - - try { - authorization = - classLoader.withClassLoader( - cl -> { - try { - ServiceLoader loader = - ServiceLoader.load(AuthorizationProvider.class, cl); - - List> providers = - Streams.stream(loader.iterator()) - .filter(p -> p.shortName().equalsIgnoreCase(authorizationProvider)) - .map(AuthorizationProvider::getClass) - .collect(Collectors.toList()); - if (providers.isEmpty()) { - throw new IllegalArgumentException( - "No authorization provider found for: " + authorizationProvider); - } else if (providers.size() > 1) { - throw new IllegalArgumentException( - "Multiple authorization providers found for: " + authorizationProvider); - } - return (BaseAuthorization) - Iterables.getOnlyElement(providers).getDeclaredConstructor().newInstance(); - } catch (Exception e) { - LOG.error("Failed to create authorization instance", e); - throw new RuntimeException(e); - } - }); - } catch (Exception e) { - LOG.error("Failed to load authorization with class loader", e); - throw new RuntimeException(e); + if (authorizationPlugin == null) { + synchronized (this) { + if (authorizationPlugin == null) { + String authorizationProvider = + (String) catalogPropertiesMetadata().getOrDefault(conf, AUTHORIZATION_PROVIDER); + if (authorizationProvider == null) { + LOG.info("Authorization provider is not set!"); + return; + } + try { + BaseAuthorization authorization = + classLoader.withClassLoader( + cl -> { + try { + ServiceLoader loader = + ServiceLoader.load(AuthorizationProvider.class, cl); + + List> providers = + Streams.stream(loader.iterator()) + .filter(p -> p.shortName().equalsIgnoreCase(authorizationProvider)) + .map(AuthorizationProvider::getClass) + .collect(Collectors.toList()); + if (providers.isEmpty()) { + throw new IllegalArgumentException( + "No authorization provider found for: " + authorizationProvider); + } else if (providers.size() > 1) { + throw new IllegalArgumentException( + "Multiple authorization providers found for: " + + authorizationProvider); + } + return (BaseAuthorization) + Iterables.getOnlyElement(providers) + .getDeclaredConstructor() + .newInstance(); + } catch (Exception e) { + LOG.error("Failed to create authorization instance", e); + throw new RuntimeException(e); + } + }); + authorizationPlugin = + authorization.newPlugin(entity.namespace().level(0), provider(), this.conf); + } catch (Exception e) { + LOG.error("Failed to load authorization with class loader", e); + throw new RuntimeException(e); + } + } + } } } @@ -244,9 +254,9 @@ public void close() throws IOException { ops.close(); ops = null; } - if (authorization != null) { - authorization.close(); - authorization = null; + if (authorizationPlugin != null) { + authorizationPlugin.close(); + authorizationPlugin = null; } } diff --git a/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java b/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java index ce460e675e1..173ad3527a8 100644 --- a/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java +++ b/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java @@ -33,7 +33,6 @@ */ public abstract class BaseAuthorization implements AuthorizationProvider, Closeable { - private volatile AuthorizationPlugin plugin = null; /** * Creates a new instance of AuthorizationPlugin.
@@ -42,26 +41,9 @@ public abstract class BaseAuthorization * * @return A new instance of AuthorizationHook. */ - protected abstract AuthorizationPlugin newPlugin( + public abstract AuthorizationPlugin newPlugin( String metalake, String catalogProvider, Map config); - public AuthorizationPlugin plugin( - String metalake, String catalogProvider, Map config) { - if (plugin == null) { - synchronized (this) { - if (plugin == null) { - plugin = newPlugin(metalake, catalogProvider, config); - } - } - } - - return plugin; - } - @Override - public void close() throws IOException { - if (plugin != null) { - plugin.close(); - } - } + public void close() throws IOException {} } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java b/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java index db7c629bbd5..e8d747da11f 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java @@ -32,7 +32,7 @@ public String shortName() { } @Override - protected AuthorizationPlugin newPlugin( + public AuthorizationPlugin newPlugin( String metalake, String catalogProvider, Map config) { return new TestMySQLAuthorizationPlugin(); } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java index 383339d0847..9df9a8d63b7 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java @@ -32,7 +32,7 @@ public String shortName() { } @Override - protected AuthorizationPlugin newPlugin( + public AuthorizationPlugin newPlugin( String metalake, String catalogProvider, Map config) { return new TestRangerAuthorizationPlugin(); } From 5e4a2ad90b38d91400d23529250dd24d627ece9d Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Fri, 20 Dec 2024 17:40:15 +1100 Subject: [PATCH 09/47] [Minor] fix comments to have correct entity (#5931) ### What changes were proposed in this pull request? fix comments to have correct entity ### Why are the changes needed? fix copy and paste error Fix: # N/A ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? N/A --- .../org/apache/gravitino/cli/commands/UpdateCatalogName.java | 2 +- .../org/apache/gravitino/cli/commands/UpdateFilesetName.java | 2 +- .../java/org/apache/gravitino/cli/commands/UpdateTableName.java | 2 +- .../java/org/apache/gravitino/cli/commands/UpdateTagName.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogName.java index 399d600fcef..8d4fcb60b96 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateCatalogName.java @@ -39,7 +39,7 @@ public class UpdateCatalogName extends Command { * @param ignoreVersions If true don't check the client/server versions match. * @param metalake The name of the metalake. * @param catalog The name of the catalog. - * @param name The new metalake name. + * @param name The new catalog name. */ public UpdateCatalogName( String url, boolean ignoreVersions, String metalake, String catalog, String name) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetName.java index 6d4ca8e0f27..a613c1f9d9b 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateFilesetName.java @@ -46,7 +46,7 @@ public class UpdateFilesetName extends Command { * @param catalog The name of the catalog. * @param schema The name of the schema. * @param fileset The name of the fileset. - * @param name The new metalake name. + * @param name The new fileset name. */ public UpdateFilesetName( String url, diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableName.java index 773d366fb3b..51a5b68722b 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTableName.java @@ -46,7 +46,7 @@ public class UpdateTableName extends Command { * @param catalog The name of the catalog. * @param schema The name of the schema. * @param table The name of the table. - * @param name The new metalake name. + * @param name The new table name. */ public UpdateTableName( String url, diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagName.java index 96fb9d15714..f4ef43412db 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UpdateTagName.java @@ -39,7 +39,7 @@ public class UpdateTagName extends Command { * @param ignoreVersions If true don't check the client/server versions match. * @param metalake The name of the tag. * @param tag The name of the catalog. - * @param name The new metalake name. + * @param name The new tag name. */ public UpdateTagName( String url, boolean ignoreVersions, String metalake, String tag, String name) { From c2ede6901a66cb0ce8ba9bf9ca1852fd527b497e Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Fri, 20 Dec 2024 15:22:19 +0800 Subject: [PATCH 10/47] [#5828] improvement(CLI): Slightly misleading error message when user is not specified in Gravitino CLI (#5921) ### What changes were proposed in this pull request? Fix missing leading error message when user is not specified in Gravitino CLI, include all commands. ### Why are the changes needed? Fix: #5828 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? ```bash bin/gcli.sh user create --metalake demo_metalake # Missing --user option. bin/gcli.sh user details --metalake demo_metalake # Missing --user option. bin/gcli.sh user delete --metalake demo_metalake # Missing --user option. bin/gcli.sh user list --metalake demo_metalake # anonymous ``` --- .../main/java/org/apache/gravitino/cli/ErrorMessages.java | 1 + .../java/org/apache/gravitino/cli/GravitinoCommandLine.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java index 323f0fc2aed..3af83b32a38 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java @@ -30,6 +30,7 @@ public class ErrorMessages { public static final String UNKNOWN_TABLE = "Unknown table name."; public static final String MALFORMED_NAME = "Malformed entity name."; public static final String MISSING_NAME = "Missing --name option."; + public static final String MISSING_USER = "Missing --user option."; public static final String METALAKE_EXISTS = "Metalake already exists."; public static final String CATALOG_EXISTS = "Catalog already exists."; public static final String SCHEMA_EXISTS = "Schema already exists."; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 4cbfaacc5ad..50bcbebaba8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -479,6 +479,11 @@ protected void handleUserCommand() { Command.setAuthenticationMode(auth, userName); + if (user == null && !CommandActions.LIST.equals(command)) { + System.err.println(ErrorMessages.MISSING_USER); + return; + } + switch (command) { case CommandActions.DETAILS: if (line.hasOption(GravitinoOptions.AUDIT)) { From 52a299da09d0334aafd271735803bee966827b35 Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Fri, 20 Dec 2024 20:18:52 +0800 Subject: [PATCH 11/47] [#5829] improvement(CLI): Slightly misleading error message when group is not specified (#5920) ### What changes were proposed in this pull request? Slightly missing leading error message when group is not specified, it should give some hints to user. ### Why are the changes needed? Fix: #5829 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? ```bash bin/gcli.sh group create --metalake demo_metalake # Missing --group option. bin/gcli.sh group details --metalake demo_metalake # Missing --group option. bin/gcli.sh group delete --metalake demo_metalake # Missing --group option. ``` --- .../main/java/org/apache/gravitino/cli/ErrorMessages.java | 1 + .../java/org/apache/gravitino/cli/GravitinoCommandLine.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java index 3af83b32a38..3423cee07f7 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java @@ -30,6 +30,7 @@ public class ErrorMessages { public static final String UNKNOWN_TABLE = "Unknown table name."; public static final String MALFORMED_NAME = "Malformed entity name."; public static final String MISSING_NAME = "Missing --name option."; + public static final String MISSING_GROUP = "Missing --group option."; public static final String MISSING_USER = "Missing --user option."; public static final String METALAKE_EXISTS = "Metalake already exists."; public static final String CATALOG_EXISTS = "Catalog already exists."; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 50bcbebaba8..a27e3cb2a08 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -540,6 +540,11 @@ protected void handleGroupCommand() { Command.setAuthenticationMode(auth, userName); + if (group == null && !CommandActions.LIST.equals(command)) { + System.err.println(ErrorMessages.MISSING_GROUP); + return; + } + switch (command) { case CommandActions.DETAILS: if (line.hasOption(GravitinoOptions.AUDIT)) { From cb30467c21aaeacca4ae2538056f0d208d68735b Mon Sep 17 00:00:00 2001 From: roryqi Date: Fri, 20 Dec 2024 22:13:08 +0800 Subject: [PATCH 12/47] [#5934] fix(auth): Avoid other catalogs' privileges are pushed down (#5935) ### What changes were proposed in this pull request? Avoid other catalogs' privileges are pushed down. For example, if a role has two catalogs. One catalog has select table, the other catalog has create table. The plugin will make the role can create and select table at the same time. ### Why are the changes needed? Fix: #5934 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add a UT --- .../authorization/AuthorizationUtils.java | 118 ++++++++++++------ .../authorization/PermissionManager.java | 83 +++++++----- .../gravitino/authorization/RoleManager.java | 11 +- .../authorization/TestAuthorizationUtils.java | 61 +++++++++ 4 files changed, 200 insertions(+), 73 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java index 793b478eb6d..61aa86f425e 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java @@ -18,10 +18,12 @@ */ package org.apache.gravitino.authorization; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.util.Collection; import java.util.List; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; @@ -39,6 +41,7 @@ import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.exceptions.NoSuchUserException; +import org.apache.gravitino.meta.RoleEntity; import org.apache.gravitino.utils.MetadataObjectUtil; import org.apache.gravitino.utils.NameIdentifierUtil; @@ -144,8 +147,8 @@ public static void checkRoleNamespace(Namespace namespace) { public static void callAuthorizationPluginForSecurableObjects( String metalake, List securableObjects, - Set catalogsAlreadySet, - Consumer consumer) { + BiConsumer consumer) { + Set catalogsAlreadySet = Sets.newHashSet(); CatalogManager catalogManager = GravitinoEnv.getInstance().catalogManager(); for (SecurableObject securableObject : securableObjects) { if (needApplyAuthorizationPluginAllCatalogs(securableObject)) { @@ -245,40 +248,6 @@ public static void checkPrivilege( } } - private static void checkCatalogType( - NameIdentifier catalogIdent, Catalog.Type type, Privilege privilege) { - Catalog catalog = GravitinoEnv.getInstance().catalogDispatcher().loadCatalog(catalogIdent); - if (catalog.type() != type) { - throw new IllegalPrivilegeException( - "Catalog %s type %s doesn't support privilege %s", - catalogIdent, catalog.type(), privilege); - } - } - - private static boolean needApplyAuthorizationPluginAllCatalogs(MetadataObject.Type type) { - return type == MetadataObject.Type.METALAKE; - } - - private static boolean needApplyAuthorization(MetadataObject.Type type) { - return type != MetadataObject.Type.ROLE && type != MetadataObject.Type.METALAKE; - } - - private static void callAuthorizationPluginImpl( - Consumer consumer, Catalog catalog) { - - if (catalog instanceof BaseCatalog) { - BaseCatalog baseCatalog = (BaseCatalog) catalog; - if (baseCatalog.getAuthorizationPlugin() != null) { - consumer.accept(baseCatalog.getAuthorizationPlugin()); - } - } else { - throw new IllegalArgumentException( - String.format( - "Catalog %s is not a BaseCatalog, we don't support authorization plugin for it", - catalog.type())); - } - } - public static void authorizationPluginRemovePrivileges( NameIdentifier ident, Entity.EntityType type) { // If we enable authorization, we should remove the privileges about the entity in the @@ -313,4 +282,81 @@ public static void authorizationPluginRenamePrivileges( }); } } + + public static Role filterSecurableObjects( + RoleEntity role, String metalakeName, String catalogName) { + List securableObjects = role.securableObjects(); + List filteredSecurableObjects = Lists.newArrayList(); + for (SecurableObject securableObject : securableObjects) { + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalakeName, securableObject); + if (securableObject.type() == MetadataObject.Type.METALAKE) { + filteredSecurableObjects.add(securableObject); + } else { + NameIdentifier catalogIdent = NameIdentifierUtil.getCatalogIdentifier(identifier); + + if (catalogIdent.name().equals(catalogName)) { + filteredSecurableObjects.add(securableObject); + } + } + } + + return RoleEntity.builder() + .withId(role.id()) + .withName(role.name()) + .withAuditInfo(role.auditInfo()) + .withNamespace(role.namespace()) + .withSecurableObjects(filteredSecurableObjects) + .withProperties(role.properties()) + .build(); + } + + private static boolean needApplyAuthorizationPluginAllCatalogs(MetadataObject.Type type) { + return type == MetadataObject.Type.METALAKE; + } + + private static boolean needApplyAuthorization(MetadataObject.Type type) { + return type != MetadataObject.Type.ROLE && type != MetadataObject.Type.METALAKE; + } + + private static void callAuthorizationPluginImpl( + BiConsumer consumer, Catalog catalog) { + + if (catalog instanceof BaseCatalog) { + BaseCatalog baseCatalog = (BaseCatalog) catalog; + if (baseCatalog.getAuthorizationPlugin() != null) { + consumer.accept(baseCatalog.getAuthorizationPlugin(), catalog.name()); + } + } else { + throw new IllegalArgumentException( + String.format( + "Catalog %s is not a BaseCatalog, we don't support authorization plugin for it", + catalog.type())); + } + } + + private static void callAuthorizationPluginImpl( + Consumer consumer, Catalog catalog) { + + if (catalog instanceof BaseCatalog) { + BaseCatalog baseCatalog = (BaseCatalog) catalog; + if (baseCatalog.getAuthorizationPlugin() != null) { + consumer.accept(baseCatalog.getAuthorizationPlugin()); + } + } else { + throw new IllegalArgumentException( + String.format( + "Catalog %s is not a BaseCatalog, we don't support authorization plugin for it", + catalog.type())); + } + } + + private static void checkCatalogType( + NameIdentifier catalogIdent, Catalog.Type type, Privilege privilege) { + Catalog catalog = GravitinoEnv.getInstance().catalogDispatcher().loadCatalog(catalogIdent); + if (catalog.type() != type) { + throw new IllegalPrivilegeException( + "Catalog %s type %s doesn't support privilege %s", + catalogIdent, catalog.type(), privilege); + } + } } diff --git a/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java b/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java index 02c240f30a9..bdaa8f6f74d 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java @@ -21,6 +21,7 @@ import static org.apache.gravitino.authorization.AuthorizationUtils.GROUP_DOES_NOT_EXIST_MSG; import static org.apache.gravitino.authorization.AuthorizationUtils.ROLE_DOES_NOT_EXIST_MSG; import static org.apache.gravitino.authorization.AuthorizationUtils.USER_DOES_NOT_EXIST_MSG; +import static org.apache.gravitino.authorization.AuthorizationUtils.filterSecurableObjects; import com.google.common.collect.Lists; import java.io.IOException; @@ -115,17 +116,22 @@ User grantRolesToUser(String metalake, List roles, String user) { .build(); }); - Set catalogs = Sets.newHashSet(); + List securableObjects = Lists.newArrayList(); + for (Role grantedRole : roleEntitiesToGrant) { - AuthorizationUtils.callAuthorizationPluginForSecurableObjects( - metalake, - grantedRole.securableObjects(), - catalogs, - authorizationPlugin -> - authorizationPlugin.onGrantedRolesToUser( - Lists.newArrayList(roleEntitiesToGrant), updatedUser)); + securableObjects.addAll(grantedRole.securableObjects()); } + AuthorizationUtils.callAuthorizationPluginForSecurableObjects( + metalake, + securableObjects, + (authorizationPlugin, catalogName) -> + authorizationPlugin.onGrantedRolesToUser( + roleEntitiesToGrant.stream() + .map(roleEntity -> filterSecurableObjects(roleEntity, metalake, catalogName)) + .collect(Collectors.toList()), + updatedUser)); + return updatedUser; } catch (NoSuchEntityException nse) { LOG.warn("Failed to grant, user {} does not exist in the metalake {}", user, metalake, nse); @@ -196,17 +202,22 @@ Group grantRolesToGroup(String metalake, List roles, String group) { .build(); }); - Set catalogs = Sets.newHashSet(); + List securableObjects = Lists.newArrayList(); + for (Role grantedRole : roleEntitiesToGrant) { - AuthorizationUtils.callAuthorizationPluginForSecurableObjects( - metalake, - grantedRole.securableObjects(), - catalogs, - authorizationPlugin -> - authorizationPlugin.onGrantedRolesToGroup( - Lists.newArrayList(roleEntitiesToGrant), updatedGroup)); + securableObjects.addAll(grantedRole.securableObjects()); } + AuthorizationUtils.callAuthorizationPluginForSecurableObjects( + metalake, + securableObjects, + (authorizationPlugin, catalogName) -> + authorizationPlugin.onGrantedRolesToGroup( + roleEntitiesToGrant.stream() + .map(roleEntity -> filterSecurableObjects(roleEntity, metalake, catalogName)) + .collect(Collectors.toList()), + updatedGroup)); + return updatedGroup; } catch (NoSuchEntityException nse) { LOG.warn("Failed to grant, group {} does not exist in the metalake {}", group, metalake, nse); @@ -276,17 +287,21 @@ Group revokeRolesFromGroup(String metalake, List roles, String group) { .build(); }); - Set catalogs = Sets.newHashSet(); + List securableObjects = Lists.newArrayList(); for (Role grantedRole : roleEntitiesToRevoke) { - AuthorizationUtils.callAuthorizationPluginForSecurableObjects( - metalake, - grantedRole.securableObjects(), - catalogs, - authorizationPlugin -> - authorizationPlugin.onRevokedRolesFromGroup( - Lists.newArrayList(roleEntitiesToRevoke), updatedGroup)); + securableObjects.addAll(grantedRole.securableObjects()); } + AuthorizationUtils.callAuthorizationPluginForSecurableObjects( + metalake, + securableObjects, + (authorizationPlugin, catalogName) -> + authorizationPlugin.onRevokedRolesFromGroup( + roleEntitiesToRevoke.stream() + .map(roleEntity -> filterSecurableObjects(roleEntity, metalake, catalogName)) + .collect(Collectors.toList()), + updatedGroup)); + return updatedGroup; } catch (NoSuchEntityException nse) { @@ -358,17 +373,21 @@ User revokeRolesFromUser(String metalake, List roles, String user) { .build(); }); - Set catalogs = Sets.newHashSet(); + List securableObjects = Lists.newArrayList(); for (Role grantedRole : roleEntitiesToRevoke) { - AuthorizationUtils.callAuthorizationPluginForSecurableObjects( - metalake, - grantedRole.securableObjects(), - catalogs, - authorizationPlugin -> - authorizationPlugin.onRevokedRolesFromUser( - Lists.newArrayList(roleEntitiesToRevoke), updatedUser)); + securableObjects.addAll(grantedRole.securableObjects()); } + AuthorizationUtils.callAuthorizationPluginForSecurableObjects( + metalake, + securableObjects, + (authorizationPlugin, catalogName) -> + authorizationPlugin.onRevokedRolesFromUser( + roleEntitiesToRevoke.stream() + .map(roleEntity -> filterSecurableObjects(roleEntity, metalake, catalogName)) + .collect(Collectors.toList()), + updatedUser)); + return updatedUser; } catch (NoSuchEntityException nse) { LOG.warn("Failed to revoke, user {} does not exist in the metalake {}", user, metalake, nse); diff --git a/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java b/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java index 11c24102bca..16e1cdda379 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java @@ -21,7 +21,6 @@ import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; -import com.google.common.collect.Sets; import java.io.IOException; import java.time.Instant; import java.util.List; @@ -87,8 +86,9 @@ RoleEntity createRole( AuthorizationUtils.callAuthorizationPluginForSecurableObjects( metalake, roleEntity.securableObjects(), - Sets.newHashSet(), - authorizationPlugin -> authorizationPlugin.onRoleCreated(roleEntity)); + (authorizationPlugin, catalogName) -> + authorizationPlugin.onRoleCreated( + AuthorizationUtils.filterSecurableObjects(roleEntity, metalake, catalogName))); return roleEntity; } catch (EntityAlreadyExistsException e) { @@ -122,8 +122,9 @@ boolean deleteRole(String metalake, String role) { AuthorizationUtils.callAuthorizationPluginForSecurableObjects( metalake, roleEntity.securableObjects(), - Sets.newHashSet(), - authorizationPlugin -> authorizationPlugin.onRoleDeleted(roleEntity)); + (authorizationPlugin, catalogName) -> + authorizationPlugin.onRoleDeleted( + AuthorizationUtils.filterSecurableObjects(roleEntity, metalake, catalogName))); } catch (NoSuchEntityException nse) { // ignore, because the role may have been deleted. } diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java b/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java index c2d844fbd86..b602471c4d1 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java @@ -18,10 +18,14 @@ */ package org.apache.gravitino.authorization; +import com.google.common.collect.Lists; +import java.util.List; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.IllegalNameIdentifierException; import org.apache.gravitino.exceptions.IllegalNamespaceException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.RoleEntity; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -149,4 +153,61 @@ void testCheckNamespace() { IllegalNamespaceException.class, () -> AuthorizationUtils.checkRoleNamespace(Namespace.of("a", "b", "c", "d"))); } + + @Test + void testFilteredSecurableObjects() { + + List securableObjects = Lists.newArrayList(); + + SecurableObject metalakeObject = + SecurableObjects.ofMetalake("metalake", Lists.newArrayList(Privileges.SelectTable.allow())); + securableObjects.add(metalakeObject); + + SecurableObject catalog1Object = + SecurableObjects.ofCatalog("catalog1", Lists.newArrayList(Privileges.SelectTable.allow())); + securableObjects.add(catalog1Object); + + SecurableObject catalog2Object = + SecurableObjects.ofCatalog("catalog2", Lists.newArrayList(Privileges.SelectTable.allow())); + securableObjects.add(catalog2Object); + + SecurableObject schema1Object = + SecurableObjects.ofSchema( + catalog1Object, "schema1", Lists.newArrayList(Privileges.SelectTable.allow())); + SecurableObject table1Object = + SecurableObjects.ofTable( + schema1Object, "table1", Lists.newArrayList(Privileges.SelectTable.allow())); + securableObjects.add(table1Object); + securableObjects.add(schema1Object); + + SecurableObject schema2Object = + SecurableObjects.ofSchema( + catalog2Object, "schema2", Lists.newArrayList(Privileges.SelectTable.allow())); + SecurableObject table2Object = + SecurableObjects.ofTable( + schema2Object, "table2", Lists.newArrayList(Privileges.SelectTable.allow())); + securableObjects.add(table2Object); + securableObjects.add(schema2Object); + + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName("role") + .withSecurableObjects(securableObjects) + .withAuditInfo(AuditInfo.EMPTY) + .build(); + Role filteredRole = AuthorizationUtils.filterSecurableObjects(role, "metalake", "catalog1"); + Assertions.assertEquals(4, filteredRole.securableObjects().size()); + Assertions.assertTrue(filteredRole.securableObjects().contains(metalakeObject)); + Assertions.assertTrue(filteredRole.securableObjects().contains(catalog1Object)); + Assertions.assertTrue(filteredRole.securableObjects().contains(schema1Object)); + Assertions.assertTrue(filteredRole.securableObjects().contains(table1Object)); + + filteredRole = AuthorizationUtils.filterSecurableObjects(role, "metalake", "catalog2"); + Assertions.assertEquals(4, filteredRole.securableObjects().size()); + Assertions.assertTrue(filteredRole.securableObjects().contains(metalakeObject)); + Assertions.assertTrue(filteredRole.securableObjects().contains(catalog2Object)); + Assertions.assertTrue(filteredRole.securableObjects().contains(schema2Object)); + Assertions.assertTrue(filteredRole.securableObjects().contains(table2Object)); + } } From f933f383f968a800d667940e5df3a97e9693fcac Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 22 Dec 2024 11:32:16 +0800 Subject: [PATCH 13/47] [#5894] feat(iceberg): support Azure account key credential (#5938) ### What changes were proposed in this pull request? Support Azure account key credential ### Why are the changes needed? Fix: #5894 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Unit Test IcebergRESTADLSAccountKeyIT at iceberg 1.6.0 --- .../credential/AzureAccountKeyCredential.java | 109 ++++++++++++++++ ...org.apache.gravitino.credential.Credential | 1 + .../abs/credential/ADLSTokenProvider.java | 14 +-- .../credential/AzureAccountKeyProvider.java | 54 ++++++++ ...he.gravitino.credential.CredentialProvider | 3 +- .../credential/CredentialConstants.java | 2 + .../credential/CredentialPropertyUtils.java | 31 +++-- .../credential/TestCredentialFactory.java | 27 ++++ ...Config.java => AzureCredentialConfig.java} | 6 +- docs/iceberg-rest-service.md | 43 +++---- ...DLSIT.java => IcebergRESTADLSTokenIT.java} | 13 +- .../test/IcebergRESTAzureAccountKeyIT.java | 117 ++++++++++++++++++ 12 files changed, 374 insertions(+), 46 deletions(-) create mode 100644 api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java create mode 100644 bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java rename core/src/main/java/org/apache/gravitino/credential/config/{ADLSCredentialConfig.java => AzureCredentialConfig.java} (96%) rename iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/{IcebergRESTADLSIT.java => IcebergRESTADLSTokenIT.java} (92%) create mode 100644 iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java diff --git a/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java b/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java new file mode 100644 index 00000000000..be24d7cda0e --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; + +/** Azure account key credential. */ +public class AzureAccountKeyCredential implements Credential { + + /** Azure account key credential type. */ + public static final String AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE = "azure-account-key"; + /** Azure storage account name */ + public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME = "azure-storage-account-name"; + /** Azure storage account key */ + public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY = "azure-storage-account-key"; + + private String accountName; + private String accountKey; + + /** + * Constructs an instance of {@link AzureAccountKeyCredential}. + * + * @param accountName The Azure account name. + * @param accountKey The Azure account key. + */ + public AzureAccountKeyCredential(String accountName, String accountKey) { + validate(accountName, accountKey); + this.accountName = accountName; + this.accountKey = accountKey; + } + + /** + * This is the constructor that is used by credential factory to create an instance of credential + * according to the credential information. + */ + public AzureAccountKeyCredential() {} + + @Override + public String credentialType() { + return AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return 0; + } + + @Override + public Map credentialInfo() { + return (new ImmutableMap.Builder()) + .put(GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, accountName) + .put(GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, accountKey) + .build(); + } + + @Override + public void initialize(Map credentialInfo, long expireTimeInMS) { + String accountName = credentialInfo.get(GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME); + String accountKey = credentialInfo.get(GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY); + validate(accountName, accountKey); + this.accountName = accountName; + this.accountKey = accountKey; + } + + /** + * Get Azure account name + * + * @return The Azure account name + */ + public String accountName() { + return accountName; + } + + /** + * Get Azure account key + * + * @return The Azure account key + */ + public String accountKey() { + return accountKey; + } + + private void validate(String accountName, String accountKey) { + Preconditions.checkArgument( + StringUtils.isNotBlank(accountName), "Azure account name should not be empty."); + Preconditions.checkArgument( + StringUtils.isNotBlank(accountKey), "Azure account key should not be empty."); + } +} diff --git a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential index f130b4b6423..6071cb916ae 100644 --- a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential +++ b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential @@ -23,3 +23,4 @@ org.apache.gravitino.credential.GCSTokenCredential org.apache.gravitino.credential.OSSTokenCredential org.apache.gravitino.credential.OSSSecretKeyCredential org.apache.gravitino.credential.ADLSTokenCredential +org.apache.gravitino.credential.AzureAccountKeyCredential diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java index e2ee3ed82a3..c2b684acbde 100644 --- a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java +++ b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java @@ -38,7 +38,7 @@ import org.apache.gravitino.credential.CredentialContext; import org.apache.gravitino.credential.CredentialProvider; import org.apache.gravitino.credential.PathBasedCredentialContext; -import org.apache.gravitino.credential.config.ADLSCredentialConfig; +import org.apache.gravitino.credential.config.AzureCredentialConfig; /** Generates ADLS token to access ADLS data. */ public class ADLSTokenProvider implements CredentialProvider { @@ -51,14 +51,14 @@ public class ADLSTokenProvider implements CredentialProvider { @Override public void initialize(Map properties) { - ADLSCredentialConfig adlsCredentialConfig = new ADLSCredentialConfig(properties); - this.storageAccountName = adlsCredentialConfig.storageAccountName(); - this.tenantId = adlsCredentialConfig.tenantId(); - this.clientId = adlsCredentialConfig.clientId(); - this.clientSecret = adlsCredentialConfig.clientSecret(); + AzureCredentialConfig azureCredentialConfig = new AzureCredentialConfig(properties); + this.storageAccountName = azureCredentialConfig.storageAccountName(); + this.tenantId = azureCredentialConfig.tenantId(); + this.clientId = azureCredentialConfig.clientId(); + this.clientSecret = azureCredentialConfig.clientSecret(); this.endpoint = String.format("https://%s.%s", storageAccountName, ADLSTokenCredential.ADLS_DOMAIN); - this.tokenExpireSecs = adlsCredentialConfig.tokenExpireInSecs(); + this.tokenExpireSecs = azureCredentialConfig.adlsTokenExpireInSecs(); } @Override diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java new file mode 100644 index 00000000000..726c4f2d996 --- /dev/null +++ b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.abs.credential; + +import java.util.Map; +import org.apache.gravitino.credential.AzureAccountKeyCredential; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.CredentialContext; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.config.AzureCredentialConfig; + +/** Generates Azure account key to access data. */ +public class AzureAccountKeyProvider implements CredentialProvider { + private String accountName; + private String accountKey; + + @Override + public void initialize(Map properties) { + AzureCredentialConfig azureCredentialConfig = new AzureCredentialConfig(properties); + this.accountName = azureCredentialConfig.storageAccountName(); + this.accountKey = azureCredentialConfig.storageAccountKey(); + } + + @Override + public void close() {} + + @Override + public String credentialType() { + return CredentialConstants.AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE; + } + + @Override + public Credential getCredential(CredentialContext context) { + return new AzureAccountKeyCredential(accountName, accountKey); + } +} diff --git a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider index fb53efffa63..4c7e7982cb1 100644 --- a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider +++ b/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -16,4 +16,5 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.abs.credential.ADLSTokenProvider \ No newline at end of file +org.apache.gravitino.abs.credential.ADLSTokenProvider +org.apache.gravitino.abs.credential.AzureAccountKeyProvider \ No newline at end of file diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index 7dd74d08484..29f9241c890 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -32,5 +32,7 @@ public class CredentialConstants { public static final String ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE = "adls-token"; public static final String ADLS_TOKEN_EXPIRE_IN_SECS = "adls-token-expire-in-secs"; + public static final String AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE = "azure-account-key"; + private CredentialConstants() {} } diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java index e1803a6ddf1..d7a3caf067f 100644 --- a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java @@ -33,12 +33,19 @@ public class CredentialPropertyUtils { @VisibleForTesting static final String ICEBERG_S3_SECRET_ACCESS_KEY = "s3.secret-access-key"; @VisibleForTesting static final String ICEBERG_S3_TOKEN = "s3.session-token"; @VisibleForTesting static final String ICEBERG_GCS_TOKEN = "gcs.oauth2.token"; - @VisibleForTesting static final String ICEBERG_ADLS_TOKEN = "adls.sas-token"; @VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_ID = "client.access-key-id"; @VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_SECRET = "client.access-key-secret"; @VisibleForTesting static final String ICEBERG_OSS_SECURITY_TOKEN = "client.security-token"; + @VisibleForTesting static final String ICEBERG_ADLS_TOKEN = "adls.sas-token"; + + @VisibleForTesting + static final String ICEBERG_ADLS_ACCOUNT_NAME = "adls.auth.shared-key.account.name"; + + @VisibleForTesting + static final String ICEBERG_ADLS_ACCOUNT_KEY = "adls.auth.shared-key.account.key"; + private static Map icebergCredentialPropertyMap = ImmutableMap.of( GCSTokenCredential.GCS_TOKEN_NAME, @@ -54,7 +61,11 @@ public class CredentialPropertyUtils { OSSTokenCredential.GRAVITINO_OSS_SESSION_ACCESS_KEY_ID, ICEBERG_OSS_ACCESS_KEY_ID, OSSTokenCredential.GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY, - ICEBERG_OSS_ACCESS_KEY_SECRET); + ICEBERG_OSS_ACCESS_KEY_SECRET, + AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, + ICEBERG_ADLS_ACCOUNT_NAME, + AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, + ICEBERG_ADLS_ACCOUNT_KEY); /** * Transforms a specific credential into a map of Iceberg properties. @@ -63,6 +74,14 @@ public class CredentialPropertyUtils { * @return a map of Iceberg properties derived from the credential */ public static Map toIcebergProperties(Credential credential) { + if (credential instanceof S3TokenCredential + || credential instanceof S3SecretKeyCredential + || credential instanceof OSSTokenCredential + || credential instanceof OSSSecretKeyCredential + || credential instanceof AzureAccountKeyCredential) { + return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); + } + if (credential instanceof GCSTokenCredential) { Map icebergGCSCredentialProperties = transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); @@ -70,12 +89,7 @@ public static Map toIcebergProperties(Credential credential) { "gcs.oauth2.token-expires-at", String.valueOf(credential.expireTimeInMs())); return icebergGCSCredentialProperties; } - if (credential instanceof S3TokenCredential || credential instanceof S3SecretKeyCredential) { - return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); - } - if (credential instanceof OSSTokenCredential || credential instanceof OSSSecretKeyCredential) { - return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); - } + if (credential instanceof ADLSTokenCredential) { ADLSTokenCredential adlsCredential = (ADLSTokenCredential) credential; String sasTokenKey = @@ -87,6 +101,7 @@ public static Map toIcebergProperties(Credential credential) { icebergADLSCredentialProperties.put(sasTokenKey, adlsCredential.sasToken()); return icebergADLSCredentialProperties; } + return credential.toProperties(); } diff --git a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java index 75a669e3887..6291b8857d7 100644 --- a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java +++ b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java @@ -165,4 +165,31 @@ void testADLSTokenCredential() { Assertions.assertEquals(sasToken, adlsTokenCredential.sasToken()); Assertions.assertEquals(expireTime, adlsTokenCredential.expireTimeInMs()); } + + @Test + void testAzureAccountKeyCredential() { + String storageAccountName = "storage-account-name"; + String storageAccountKey = "storage-account-key"; + + Map azureAccountKeyCredentialInfo = + ImmutableMap.of( + AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, + storageAccountName, + AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, + storageAccountKey); + long expireTime = 0; + Credential credential = + CredentialFactory.create( + AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE, + azureAccountKeyCredentialInfo, + expireTime); + Assertions.assertEquals( + AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE, credential.credentialType()); + Assertions.assertInstanceOf(AzureAccountKeyCredential.class, credential); + + AzureAccountKeyCredential azureAccountKeyCredential = (AzureAccountKeyCredential) credential; + Assertions.assertEquals(storageAccountName, azureAccountKeyCredential.accountName()); + Assertions.assertEquals(storageAccountKey, azureAccountKeyCredential.accountKey()); + Assertions.assertEquals(expireTime, azureAccountKeyCredential.expireTimeInMs()); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java similarity index 96% rename from core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java rename to core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java index e9d368e6752..155cc6806e0 100644 --- a/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java +++ b/core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java @@ -29,7 +29,7 @@ import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.storage.AzureProperties; -public class ADLSCredentialConfig extends Config { +public class AzureCredentialConfig extends Config { public static final ConfigEntry AZURE_STORAGE_ACCOUNT_NAME = new ConfigBuilder(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME) @@ -79,7 +79,7 @@ public class ADLSCredentialConfig extends Config { .intConf() .createWithDefault(3600); - public ADLSCredentialConfig(Map properties) { + public AzureCredentialConfig(Map properties) { super(false); loadFromMap(properties, k -> true); } @@ -110,7 +110,7 @@ public String clientSecret() { } @NotNull - public Integer tokenExpireInSecs() { + public Integer adlsTokenExpireInSecs() { return this.get(ADLS_TOKEN_EXPIRE_IN_SECS); } } diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 8d9d49745c2..f31aa13685a 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -106,18 +106,18 @@ The detailed configuration items are as follows: Gravitino Iceberg REST service supports using static S3 secret key or generating temporary token to access S3 data. -| Configuration item | Description | Default value | Required | Since Version | -|---------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------|------------------| -| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.credential-provider-type` | Supports `s3-token` and `s3-secret-key` for S3. `s3-token` generates a temporary token according to the query data path while `s3-secret-key` using the s3 secret access key to access S3 data. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-access-key-id` | The static access key ID used to access S3 data. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret access key used to access S3 data. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-region` | The region of the S3 service, like `us-west-2`. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to access the S3 data. | (none) | Yes, when `credential-provider-type` is `s3-token` | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to generate token, only used when `credential-provider-type` is `s3-token`. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token expire time in secs, it couldn't exceed the max session time of the assumed role, only used when `credential-provider-type` is `s3-token`. | 3600 | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint of the S3 token service, This could be used with s3-compatible object storage service like MINIO that has a different STS endpoint. | (none) | No | 0.8.0-incubating | +| Configuration item | Description | Default value | Required | Since Version | +|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------|------------------| +| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.credential-provider-type` | Supports `s3-token` and `s3-secret-key` for S3. `s3-token` generates a temporary token according to the query data path while `s3-secret-key` using the s3 secret access key to access S3 data. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-access-key-id` | The static access key ID used to access S3 data. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret access key used to access S3 data. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-region` | The region of the S3 service, like `us-west-2`. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to access the S3 data. | (none) | Yes, when `credential-provider-type` is `s3-token` | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to generate token, only used when `credential-provider-type` is `s3-token`. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token expire time in secs, it couldn't exceed the max session time of the assumed role, only used when `credential-provider-type` is `s3-token`. | 3600 | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint of the S3 token service, This could be used with s3-compatible object storage service like MINIO that has a different STS endpoint. | (none) | No | 0.8.0-incubating | For other Iceberg s3 properties not managed by Gravitino like `s3.sse.type`, you could config it directly by `gravitino.iceberg-rest.s3.sse.type`. @@ -175,15 +175,16 @@ Please set `gravitino.iceberg-rest.warehouse` to `gs://{bucket_name}/${prefix_na Gravitino Iceberg REST service supports generating SAS token to access ADLS data. -| Configuration item | Description | Default value | Required | Since Version | -|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------------|---------------|----------|------------------| -| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for ADLS. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token`, generates a temporary token according to the query data path. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-storage-account-name` | The static storage account name used to access ADLS data. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-storage-account-key` | The static storage account key used to access ADLS data. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory (AAD) tenant ID. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory (AAD) client ID used for authentication. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory (AAD) client secret used for authentication. | (none) | Yes | 0.8.0-incubating | +| Configuration item | Description | Default value | Required | Since Version | +|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|------------------| +| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for ADLS. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token` and `azure-account-key`. `adls-token` generates a temporary token according to the query data path while `azure-account-key` uses a storage account key to access ADLS data. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-storage-account-name` | The static storage account name used to access ADLS data. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-storage-account-key` | The static storage account key used to access ADLS data. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory (AAD) tenant ID, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory (AAD) client ID used for authentication, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory (AAD) client secret used for authentication, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.adls-token-expire-in-secs` | The ADLS SAS token expire time in secs, only used when `credential-provider-type` is `adls-token`. | 3600 | No | 0.8.0-incubating | For other Iceberg ADLS properties not managed by Gravitino like `adls.read.block-size-bytes`, you could config it directly by `gravitino.iceberg-rest.adls.read.block-size-bytes`. diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java similarity index 92% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java index 570298d050b..b16d504e1ea 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java @@ -36,7 +36,7 @@ @SuppressWarnings("FormatStringAnnotation") @EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") -public class IcebergRESTADLSIT extends IcebergRESTJdbcCatalogIT { +public class IcebergRESTADLSTokenIT extends IcebergRESTJdbcCatalogIT { private String storageAccountName; private String storageAccountKey; @@ -49,13 +49,14 @@ public class IcebergRESTADLSIT extends IcebergRESTJdbcCatalogIT { void initEnv() { this.storageAccountName = System.getenv() - .getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_NAME", "{STORAGE_ACCOUNT_NAME}"); + .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME", "{STORAGE_ACCOUNT_NAME}"); this.storageAccountKey = - System.getenv().getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_KEY", "{STORAGE_ACCOUNT_KEY}"); - this.tenantId = System.getenv().getOrDefault("GRAVITINO_ADLS_TENANT_ID", "{TENANT_ID}"); - this.clientId = System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_ID", "{CLIENT_ID}"); + System.getenv() + .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY", "{STORAGE_ACCOUNT_KEY}"); + this.tenantId = System.getenv().getOrDefault("GRAVITINO_AZURE_TENANT_ID", "{TENANT_ID}"); + this.clientId = System.getenv().getOrDefault("GRAVITINO_AZURE_CLIENT_ID", "{CLIENT_ID}"); this.clientSecret = - System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_SECRET", "{CLIENT_SECRET}"); + System.getenv().getOrDefault("GRAVITINO_AZURE_CLIENT_SECRET", "{CLIENT_SECRET}"); this.warehousePath = String.format( "abfss://%s@%s.dfs.core.windows.net/data/test", diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java new file mode 100644 index 00000000000..42709162aaa --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.integration.test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.DownloaderUtils; +import org.apache.gravitino.integration.test.util.ITUtils; +import org.apache.gravitino.storage.AzureProperties; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +@SuppressWarnings("FormatStringAnnotation") +@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") +public class IcebergRESTAzureAccountKeyIT extends IcebergRESTJdbcCatalogIT { + + private String storageAccountName; + private String storageAccountKey; + private String warehousePath; + + @Override + void initEnv() { + this.storageAccountName = + System.getenv() + .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME", "{STORAGE_ACCOUNT_NAME}"); + this.storageAccountKey = + System.getenv() + .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY", "{STORAGE_ACCOUNT_KEY}"); + this.warehousePath = + String.format( + "abfss://%s@%s.dfs.core.windows.net/data/test", + System.getenv().getOrDefault("GRAVITINO_ADLS_CONTAINER", "{ADLS_CONTAINER}"), + storageAccountName); + + if (ITUtils.isEmbedded()) { + return; + } + try { + downloadIcebergAzureBundleJar(); + } catch (IOException e) { + LOG.warn("Download Iceberg Azure bundle jar failed,", e); + throw new RuntimeException(e); + } + copyAzureBundleJar(); + } + + @Override + public Map getCatalogConfig() { + HashMap m = new HashMap(); + m.putAll(getCatalogJdbcConfig()); + m.putAll(getADLSConfig()); + return m; + } + + public boolean supportsCredentialVending() { + return true; + } + + private Map getADLSConfig() { + Map configMap = new HashMap(); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + CredentialConstants.AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, + storageAccountName); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, + storageAccountKey); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.IO_IMPL, + "org.apache.iceberg.azure.adlsv2.ADLSFileIO"); + configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.WAREHOUSE, warehousePath); + + return configMap; + } + + private void downloadIcebergAzureBundleJar() throws IOException { + String icebergBundleJarName = "iceberg-azure-bundle-1.5.2.jar"; + String icebergBundleJarUri = + "https://repo1.maven.org/maven2/org/apache/iceberg/" + + "iceberg-azure-bundle/1.5.2/" + + icebergBundleJarName; + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir); + } + + private void copyAzureBundleJar() { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir); + } +} From ef2f92d76ce70e95cbb2d2af5738692c254f60cb Mon Sep 17 00:00:00 2001 From: fsalhi2 Date: Mon, 23 Dec 2024 04:03:41 +0100 Subject: [PATCH 14/47] [#5756] Bug Fix : Warehouse parameter systematically required (#5923) ### What changes were proposed in this pull request? Removed the systematic validations in Iceberg Config and modified the instantiation of the warehouse attribute and uri attribute in the IcebergCatalogWrapper to conform with the new possibility (REST). ### Why are the changes needed? The bug was blocking the creation of a rest catalog. Fix: [#5756](https://github.com/apache/gravitino/issues/5756) ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? ./gradlew build && compileDistribution && assembleDistribution + creation of the docker image there : https://hub.docker.com/r/fsalhi2/gravitino Started gravitino with such configs : ``` ### Gravitino General Settings gravitino.auxService.names = iceberg-rest gravitino.iceberg-rest.classpath = iceberg-rest-server/libs, iceberg-rest-server/conf ### HTTP Server gravitino.iceberg-rest.host = 0.0.0.0 gravitino.iceberg-rest.httpPort = 9001 ### Storage gravitino.iceberg-rest.io-impl = org.apache.iceberg.aws.s3.S3FileIO gravitino.iceberg-rest.s3-access-key-id = XXXXX gravitino.iceberg-rest.s3-secret-access-key = XXXXXX gravitino.iceberg-rest.s3-path-style-access = true gravitino.iceberg-rest.s3-endpoint = http://minio:9000/ gravitino.iceberg-rest.s3-region = us-east-1 ### JDBC gravitino.iceberg-rest.catalog-backend = jdbc gravitino.iceberg-rest.uri = jdbc:mysql://mysql:3306/ gravitino.iceberg-rest.warehouse = s3://lake/catalog gravitino.iceberg-rest.jdbc.user = root gravitino.iceberg-rest.jdbc.password = XXXXXX gravitino.iceberg-rest.jdbc-driver = com.mysql.cj.jdbc.Driver ``` Was able to create a catalog through Web UI and start working on the scheme. --- .../IcebergCatalogPropertiesMetadata.java | 3 +- .../lakehouse/iceberg/TestIcebergCatalog.java | 47 +++++++++++++++++++ .../iceberg/common/IcebergConfig.java | 1 - .../common/ops/IcebergCatalogWrapper.java | 10 +++- 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java index 6d61a6220a3..375edd600fb 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java @@ -74,10 +74,11 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad false /* reserved */), stringRequiredPropertyEntry( URI, "Iceberg catalog uri config", false /* immutable */, false /* hidden */), - stringRequiredPropertyEntry( + stringOptionalPropertyEntry( WAREHOUSE, "Iceberg catalog warehouse config", false /* immutable */, + null, /* defaultValue */ false /* hidden */), stringOptionalPropertyEntry( IcebergConstants.IO_IMPL, diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java index 5c657197231..8ff70d39854 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java @@ -146,4 +146,51 @@ void testCatalogProperty() { throwable.getMessage().contains(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND)); } } + + @Test + void testCatalogInstanciation() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + CatalogEntity entity = + CatalogEntity.builder() + .withId(1L) + .withName("catalog") + .withNamespace(Namespace.of("metalake")) + .withType(IcebergCatalog.Type.RELATIONAL) + .withProvider("iceberg") + .withAuditInfo(auditInfo) + .build(); + + Map conf = Maps.newHashMap(); + + try (IcebergCatalogOperations ops = new IcebergCatalogOperations()) { + ops.initialize(conf, entity.toCatalogInfo(), ICEBERG_PROPERTIES_METADATA); + Map map1 = Maps.newHashMap(); + map1.put(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND, "test"); + PropertiesMetadata metadata = ICEBERG_PROPERTIES_METADATA.catalogPropertiesMetadata(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + PropertiesMetadataHelpers.validatePropertyForCreate(metadata, map1); + }); + + Map map2 = Maps.newHashMap(); + map2.put(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND, "rest"); + map2.put(IcebergCatalogPropertiesMetadata.URI, "127.0.0.1"); + Assertions.assertDoesNotThrow( + () -> { + PropertiesMetadataHelpers.validatePropertyForCreate(metadata, map2); + }); + + Map map3 = Maps.newHashMap(); + Throwable throwable = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> PropertiesMetadataHelpers.validatePropertyForCreate(metadata, map3)); + + Assertions.assertTrue( + throwable.getMessage().contains(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND)); + } + } } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java index 2e7eb74e2f1..60a7491b854 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java @@ -65,7 +65,6 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .doc("Warehouse directory of catalog") .version(ConfigConstants.VERSION_0_2_0) .stringConf() - .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry CATALOG_URI = diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java index 05c9ee2a1eb..0ed62b26f7f 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java @@ -29,6 +29,7 @@ import java.util.function.Supplier; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergCatalogBackend; import org.apache.gravitino.iceberg.common.IcebergConfig; @@ -82,9 +83,14 @@ public IcebergCatalogWrapper(IcebergConfig icebergConfig) { this.catalogBackend = IcebergCatalogBackend.valueOf( icebergConfig.get(IcebergConfig.CATALOG_BACKEND).toUpperCase(Locale.ROOT)); - if (!IcebergCatalogBackend.MEMORY.equals(catalogBackend)) { + if (!IcebergCatalogBackend.MEMORY.equals(catalogBackend) + && !IcebergCatalogBackend.REST.equals(catalogBackend)) { // check whether IcebergConfig.CATALOG_WAREHOUSE exists - icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE); + if (StringUtils.isBlank(icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE))) { + throw new IllegalArgumentException("The 'warehouse' parameter must have a value."); + } + } + if (!IcebergCatalogBackend.MEMORY.equals(catalogBackend)) { this.catalogUri = icebergConfig.get(IcebergConfig.CATALOG_URI); } this.catalog = IcebergCatalogUtil.loadCatalogBackend(catalogBackend, icebergConfig); From b516b32ff83c417d17aeaef092d5877db006488a Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 23 Dec 2024 12:12:26 +0800 Subject: [PATCH 15/47] [#5794] feat(core): Add ModelOperationDispatcher logic (#5908) ### What changes were proposed in this pull request? This PR adds the ModelOperationDispatcher logic in core. ### Why are the changes needed? This is a part of work to support model management. Fix: #5794 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Added UTs to test. --- .../gravitino/catalog/CatalogManager.java | 15 + .../catalog/EntityCombinedFileset.java | 2 +- .../catalog/EntityCombinedModel.java | 94 +++++++ .../catalog/EntityCombinedModelVersion.java | 101 +++++++ .../catalog/EntityCombinedSchema.java | 2 +- .../catalog/EntityCombinedTable.java | 2 +- .../catalog/EntityCombinedTopic.java | 2 +- .../catalog/FilesetOperationDispatcher.java | 6 +- .../catalog/ModelOperationDispatcher.java | 166 ++++++++++- .../catalog/SchemaOperationDispatcher.java | 18 +- .../catalog/TableOperationDispatcher.java | 16 +- .../catalog/TopicOperationDispatcher.java | 10 +- .../org/apache/gravitino/TestCatalog.java | 5 + .../java/org/apache/gravitino/TestModel.java | 44 +++ .../apache/gravitino/TestModelVersion.java | 45 +++ .../catalog/TestModelOperationDispatcher.java | 264 ++++++++++++++++++ .../connector/TestCatalogOperations.java | 248 +++++++++++++++- 17 files changed, 1000 insertions(+), 40 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/catalog/EntityCombinedModel.java create mode 100644 core/src/main/java/org/apache/gravitino/catalog/EntityCombinedModelVersion.java create mode 100644 core/src/test/java/org/apache/gravitino/TestModel.java create mode 100644 core/src/test/java/org/apache/gravitino/TestModelVersion.java create mode 100644 core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index 2e77b8e162a..43bc74bb2a1 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -95,6 +95,7 @@ import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.model.ModelCatalog; import org.apache.gravitino.rel.SupportsPartitions; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.TableCatalog; @@ -178,6 +179,16 @@ public R doWithTopicOps(ThrowableFunction fn) throws Except }); } + public R doWithModelOps(ThrowableFunction fn) throws Exception { + return classLoader.withClassLoader( + cl -> { + if (asModels() == null) { + throw new UnsupportedOperationException("Catalog does not support model operations"); + } + return fn.apply(asModels()); + }); + } + public R doWithCatalogOps(ThrowableFunction fn) throws Exception { return classLoader.withClassLoader(cl -> fn.apply(catalog.ops())); } @@ -236,6 +247,10 @@ private FilesetCatalog asFilesets() { private TopicCatalog asTopics() { return catalog.ops() instanceof TopicCatalog ? (TopicCatalog) catalog.ops() : null; } + + private ModelCatalog asModels() { + return catalog.ops() instanceof ModelCatalog ? (ModelCatalog) catalog.ops() : null; + } } private final Config config; diff --git a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedFileset.java b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedFileset.java index 2a6b55a2ddd..c7b847fc9c6 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedFileset.java +++ b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedFileset.java @@ -48,7 +48,7 @@ public static EntityCombinedFileset of(Fileset fileset) { return new EntityCombinedFileset(fileset, null); } - public EntityCombinedFileset withHiddenPropertiesSet(Set hiddenProperties) { + public EntityCombinedFileset withHiddenProperties(Set hiddenProperties) { this.hiddenProperties = hiddenProperties; return this; } diff --git a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedModel.java b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedModel.java new file mode 100644 index 00000000000..4aeefa0be59 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedModel.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.gravitino.Audit; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.ModelEntity; +import org.apache.gravitino.model.Model; + +public final class EntityCombinedModel implements Model { + + private final Model model; + + private final ModelEntity modelEntity; + + private Set hiddenProperties = Collections.emptySet(); + + private EntityCombinedModel(Model model, ModelEntity modelEntity) { + this.model = model; + this.modelEntity = modelEntity; + } + + public static EntityCombinedModel of(Model model, ModelEntity modelEntity) { + return new EntityCombinedModel(model, modelEntity); + } + + public static EntityCombinedModel of(Model model) { + return new EntityCombinedModel(model, null); + } + + public EntityCombinedModel withHiddenProperties(Set hiddenProperties) { + this.hiddenProperties = hiddenProperties; + return this; + } + + @Override + public String name() { + return model.name(); + } + + @Override + public String comment() { + return model.comment(); + } + + @Override + public Map properties() { + return model.properties() == null + ? null + : model.properties().entrySet().stream() + .filter(e -> !hiddenProperties.contains(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public int latestVersion() { + return model.latestVersion(); + } + + @Override + public Audit auditInfo() { + AuditInfo mergedAudit = + AuditInfo.builder() + .withCreator(model.auditInfo().creator()) + .withCreateTime(model.auditInfo().createTime()) + .withLastModifier(model.auditInfo().lastModifier()) + .withLastModifiedTime(model.auditInfo().lastModifiedTime()) + .build(); + + return modelEntity == null + ? mergedAudit + : mergedAudit.merge(modelEntity.auditInfo(), true /* overwrite */); + } +} diff --git a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedModelVersion.java b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedModelVersion.java new file mode 100644 index 00000000000..b41e2889de3 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedModelVersion.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.gravitino.Audit; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.ModelVersionEntity; +import org.apache.gravitino.model.ModelVersion; + +public final class EntityCombinedModelVersion implements ModelVersion { + + private final ModelVersion modelVersion; + + private final ModelVersionEntity modelVersionEntity; + + private Set hiddenProperties = Collections.emptySet(); + + private EntityCombinedModelVersion( + ModelVersion modelVersion, ModelVersionEntity modelVersionEntity) { + this.modelVersion = modelVersion; + this.modelVersionEntity = modelVersionEntity; + } + + public static EntityCombinedModelVersion of( + ModelVersion modelVersion, ModelVersionEntity modelVersionEntity) { + return new EntityCombinedModelVersion(modelVersion, modelVersionEntity); + } + + public static EntityCombinedModelVersion of(ModelVersion modelVersion) { + return new EntityCombinedModelVersion(modelVersion, null); + } + + public EntityCombinedModelVersion withHiddenProperties(Set hiddenProperties) { + this.hiddenProperties = hiddenProperties; + return this; + } + + @Override + public int version() { + return modelVersion.version(); + } + + @Override + public String comment() { + return modelVersion.comment(); + } + + @Override + public Map properties() { + return modelVersion.properties() == null + ? null + : modelVersion.properties().entrySet().stream() + .filter(e -> !hiddenProperties.contains(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public String uri() { + return modelVersion.uri(); + } + + @Override + public String[] aliases() { + return modelVersion.aliases(); + } + + @Override + public Audit auditInfo() { + AuditInfo mergedAudit = + AuditInfo.builder() + .withCreator(modelVersion.auditInfo().creator()) + .withCreateTime(modelVersion.auditInfo().createTime()) + .withLastModifier(modelVersion.auditInfo().lastModifier()) + .withLastModifiedTime(modelVersion.auditInfo().lastModifiedTime()) + .build(); + + return modelVersionEntity == null + ? mergedAudit + : mergedAudit.merge(modelVersionEntity.auditInfo(), true /* overwrite */); + } +} diff --git a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedSchema.java b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedSchema.java index 79a4b12a10c..ce3d0a3be72 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedSchema.java +++ b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedSchema.java @@ -61,7 +61,7 @@ public static EntityCombinedSchema of(Schema schema) { return of(schema, null); } - public EntityCombinedSchema withHiddenPropertiesSet(Set hiddenProperties) { + public EntityCombinedSchema withHiddenProperties(Set hiddenProperties) { this.hiddenProperties = hiddenProperties; return this; } diff --git a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedTable.java b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedTable.java index 4b0da1568b9..70cbd0ace4a 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedTable.java +++ b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedTable.java @@ -67,7 +67,7 @@ public static EntityCombinedTable of(Table table) { return new EntityCombinedTable(table, null); } - public EntityCombinedTable withHiddenPropertiesSet(Set hiddenProperties) { + public EntityCombinedTable withHiddenProperties(Set hiddenProperties) { this.hiddenProperties = hiddenProperties; return this; } diff --git a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedTopic.java b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedTopic.java index 2360f31ae74..972df622b3d 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedTopic.java +++ b/core/src/main/java/org/apache/gravitino/catalog/EntityCombinedTopic.java @@ -60,7 +60,7 @@ public static EntityCombinedTopic of(Topic topic) { return new EntityCombinedTopic(topic, null); } - public EntityCombinedTopic withHiddenPropertiesSet(Set hiddenProperties) { + public EntityCombinedTopic withHiddenProperties(Set hiddenProperties) { this.hiddenProperties = hiddenProperties; return this; } diff --git a/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java index 98c6311bd7c..828e981380a 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java @@ -81,7 +81,7 @@ public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { // Currently we only support maintaining the Fileset in the Gravitino's store. return EntityCombinedFileset.of(fileset) - .withHiddenPropertiesSet( + .withHiddenProperties( getHiddenPropertyNames( catalogIdent, HasPropertyMetadata::filesetPropertiesMetadata, @@ -137,7 +137,7 @@ public Fileset createFileset( NoSuchSchemaException.class, FilesetAlreadyExistsException.class); return EntityCombinedFileset.of(createdFileset) - .withHiddenPropertiesSet( + .withHiddenProperties( getHiddenPropertyNames( catalogIdent, HasPropertyMetadata::filesetPropertiesMetadata, @@ -172,7 +172,7 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) NoSuchFilesetException.class, IllegalArgumentException.class); return EntityCombinedFileset.of(alteredFileset) - .withHiddenPropertiesSet( + .withHiddenProperties( getHiddenPropertyNames( catalogIdent, HasPropertyMetadata::filesetPropertiesMetadata, diff --git a/core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java index eb1f17c96da..1c5291d51a2 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java @@ -18,15 +18,23 @@ */ package org.apache.gravitino.catalog; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import static org.apache.gravitino.utils.NameIdentifierUtil.getCatalogIdentifier; + import java.util.Map; +import java.util.function.Supplier; import org.apache.gravitino.EntityStore; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.StringIdentifier; +import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.exceptions.ModelAlreadyExistsException; import org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException; import org.apache.gravitino.exceptions.NoSuchModelException; import org.apache.gravitino.exceptions.NoSuchModelVersionException; import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.lock.LockType; +import org.apache.gravitino.lock.TreeLockUtils; import org.apache.gravitino.model.Model; import org.apache.gravitino.model.ModelVersion; import org.apache.gravitino.storage.IdGenerator; @@ -40,40 +48,114 @@ public ModelOperationDispatcher( @Override public NameIdentifier[] listModels(Namespace namespace) throws NoSuchSchemaException { - throw new UnsupportedOperationException("Not implemented"); + return TreeLockUtils.doWithTreeLock( + NameIdentifier.of(namespace.levels()), + LockType.READ, + () -> + doWithCatalog( + getCatalogIdentifier(NameIdentifier.of(namespace.levels())), + c -> c.doWithModelOps(m -> m.listModels(namespace)), + NoSuchSchemaException.class)); } @Override public Model getModel(NameIdentifier ident) throws NoSuchModelException { - throw new UnsupportedOperationException("Not implemented"); + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + Model model = + TreeLockUtils.doWithTreeLock( + ident, + LockType.READ, + () -> + doWithCatalog( + catalogIdent, + c -> c.doWithModelOps(m -> m.getModel(ident)), + NoSuchModelException.class)); + + return EntityCombinedModel.of(model) + .withHiddenProperties( + getHiddenPropertyNames( + catalogIdent, HasPropertyMetadata::modelPropertiesMetadata, model.properties())); } @Override public Model registerModel(NameIdentifier ident, String comment, Map properties) throws NoSuchModelException, ModelAlreadyExistsException { - throw new UnsupportedOperationException("Not implemented"); + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + Map updatedProperties = checkAndUpdateProperties(catalogIdent, properties); + + Model registeredModel = + TreeLockUtils.doWithTreeLock( + NameIdentifier.of(ident.namespace().levels()), + LockType.WRITE, + () -> + doWithCatalog( + catalogIdent, + c -> c.doWithModelOps(m -> m.registerModel(ident, comment, updatedProperties)), + NoSuchSchemaException.class, + ModelAlreadyExistsException.class)); + + return EntityCombinedModel.of(registeredModel) + .withHiddenProperties( + getHiddenPropertyNames( + catalogIdent, + HasPropertyMetadata::modelPropertiesMetadata, + registeredModel.properties())); } @Override public boolean deleteModel(NameIdentifier ident) { - throw new UnsupportedOperationException("Not implemented"); + return TreeLockUtils.doWithTreeLock( + NameIdentifier.of(ident.namespace().levels()), + LockType.WRITE, + () -> + doWithCatalog( + getCatalogIdentifier(ident), + c -> c.doWithModelOps(m -> m.deleteModel(ident)), + RuntimeException.class)); } @Override public int[] listModelVersions(NameIdentifier ident) throws NoSuchModelException { - throw new UnsupportedOperationException("Not implemented"); + return TreeLockUtils.doWithTreeLock( + ident, + LockType.READ, + () -> + doWithCatalog( + getCatalogIdentifier(ident), + c -> c.doWithModelOps(m -> m.listModelVersions(ident)), + NoSuchModelException.class)); } @Override public ModelVersion getModelVersion(NameIdentifier ident, int version) throws NoSuchModelVersionException { - throw new UnsupportedOperationException("Not implemented"); + return internalGetModelVersion( + ident, + () -> + TreeLockUtils.doWithTreeLock( + ident, + LockType.READ, + () -> + doWithCatalog( + getCatalogIdentifier(ident), + c -> c.doWithModelOps(m -> m.getModelVersion(ident, version)), + NoSuchModelVersionException.class))); } @Override public ModelVersion getModelVersion(NameIdentifier ident, String alias) throws NoSuchModelVersionException { - throw new UnsupportedOperationException("Not implemented"); + return internalGetModelVersion( + ident, + () -> + TreeLockUtils.doWithTreeLock( + ident, + LockType.READ, + () -> + doWithCatalog( + getCatalogIdentifier(ident), + c -> c.doWithModelOps(m -> m.getModelVersion(ident, alias)), + NoSuchModelVersionException.class))); } @Override @@ -84,16 +166,80 @@ public void linkModelVersion( String comment, Map properties) throws NoSuchModelException, ModelVersionAliasesAlreadyExistException { - throw new UnsupportedOperationException("Not implemented"); + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + Map updatedProperties = checkAndUpdateProperties(catalogIdent, properties); + + TreeLockUtils.doWithTreeLock( + ident, + LockType.WRITE, + () -> + doWithCatalog( + catalogIdent, + c -> + c.doWithModelOps( + m -> { + m.linkModelVersion(ident, uri, aliases, comment, updatedProperties); + return null; + }), + NoSuchModelException.class, + ModelVersionAliasesAlreadyExistException.class)); } @Override public boolean deleteModelVersion(NameIdentifier ident, int version) { - throw new UnsupportedOperationException("Not implemented"); + return TreeLockUtils.doWithTreeLock( + ident, + LockType.WRITE, + () -> + doWithCatalog( + getCatalogIdentifier(ident), + c -> c.doWithModelOps(m -> m.deleteModelVersion(ident, version)), + RuntimeException.class)); } @Override public boolean deleteModelVersion(NameIdentifier ident, String alias) { - throw new UnsupportedOperationException("Not implemented"); + return TreeLockUtils.doWithTreeLock( + ident, + LockType.WRITE, + () -> + doWithCatalog( + getCatalogIdentifier(ident), + c -> c.doWithModelOps(m -> m.deleteModelVersion(ident, alias)), + RuntimeException.class)); + } + + private ModelVersion internalGetModelVersion( + NameIdentifier ident, Supplier supplier) { + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + + ModelVersion modelVersion = supplier.get(); + return EntityCombinedModelVersion.of(modelVersion) + .withHiddenProperties( + getHiddenPropertyNames( + catalogIdent, + HasPropertyMetadata::modelPropertiesMetadata, + modelVersion.properties())); + } + + private Map checkAndUpdateProperties( + NameIdentifier catalogIdent, Map properties) { + TreeLockUtils.doWithTreeLock( + catalogIdent, + LockType.READ, + () -> + doWithCatalog( + catalogIdent, + c -> + c.doWithPropertiesMeta( + p -> { + validatePropertyForCreate(p.modelPropertiesMetadata(), properties); + return null; + }), + IllegalArgumentException.class)); + + long uid = idGenerator.nextId(); + StringIdentifier stringId = StringIdentifier.fromId(uid); + return StringIdentifier.newPropertiesWithId(stringId, properties); } } diff --git a/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java index ce870523a14..789e5e47155 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java @@ -125,7 +125,7 @@ public Schema createSchema(NameIdentifier ident, String comment, Map { + + private Builder() {} + + @Override + protected TestModel internalBuild() { + TestModel model = new TestModel(); + model.name = name; + model.comment = comment; + model.properties = properties; + model.latestVersion = latestVersion; + model.auditInfo = auditInfo; + return model; + } + } + + public static Builder builder() { + return new Builder(); + } +} diff --git a/core/src/test/java/org/apache/gravitino/TestModelVersion.java b/core/src/test/java/org/apache/gravitino/TestModelVersion.java new file mode 100644 index 00000000000..487496c5fb0 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/TestModelVersion.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino; + +import org.apache.gravitino.connector.BaseModelVersion; + +public class TestModelVersion extends BaseModelVersion { + + public static class Builder extends BaseModelVersionBuilder { + + private Builder() {} + + @Override + protected TestModelVersion internalBuild() { + TestModelVersion modelVersion = new TestModelVersion(); + modelVersion.version = version; + modelVersion.comment = comment; + modelVersion.aliases = aliases; + modelVersion.uri = uri; + modelVersion.properties = properties; + modelVersion.auditInfo = auditInfo; + return modelVersion; + } + } + + public static Builder builder() { + return new Builder(); + } +} diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java new file mode 100644 index 00000000000..10bb85a1e11 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog; + +import static org.apache.gravitino.Configs.TREE_LOCK_CLEAN_INTERVAL; +import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; +import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; +import static org.apache.gravitino.StringIdentifier.ID_KEY; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.gravitino.Config; +import org.apache.gravitino.GravitinoEnv; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.exceptions.NoSuchModelException; +import org.apache.gravitino.exceptions.NoSuchModelVersionException; +import org.apache.gravitino.lock.LockManager; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestModelOperationDispatcher extends TestOperationDispatcher { + + static ModelOperationDispatcher modelOperationDispatcher; + + static SchemaOperationDispatcher schemaOperationDispatcher; + + @BeforeAll + public static void initialize() throws IOException, IllegalAccessException { + Config config = Mockito.mock(Config.class); + Mockito.doReturn(100000L).when(config).get(TREE_LOCK_MAX_NODE_IN_MEMORY); + Mockito.doReturn(1000L).when(config).get(TREE_LOCK_MIN_NODE_IN_MEMORY); + Mockito.doReturn(36000L).when(config).get(TREE_LOCK_CLEAN_INTERVAL); + FieldUtils.writeField(GravitinoEnv.getInstance(), "lockManager", new LockManager(config), true); + + modelOperationDispatcher = + new ModelOperationDispatcher(catalogManager, entityStore, idGenerator); + schemaOperationDispatcher = + new SchemaOperationDispatcher(catalogManager, entityStore, idGenerator); + } + + @Test + public void testRegisterAndGetModel() { + String schemaName = randomSchemaName(); + NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, schemaName); + schemaOperationDispatcher.createSchema(schemaIdent, "comment", null); + + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + String modelName = randomModelName(); + NameIdentifier modelIdent = + NameIdentifierUtil.ofModel(metalake, catalog, schemaName, modelName); + + Model model = modelOperationDispatcher.registerModel(modelIdent, "comment", props); + Assertions.assertEquals(modelName, model.name()); + Assertions.assertEquals("comment", model.comment()); + Assertions.assertEquals(props, model.properties()); + Assertions.assertFalse(model.properties().containsKey(ID_KEY)); + + Model registeredModel = modelOperationDispatcher.getModel(modelIdent); + Assertions.assertEquals(modelName, registeredModel.name()); + Assertions.assertEquals("comment", registeredModel.comment()); + Assertions.assertEquals(props, registeredModel.properties()); + Assertions.assertFalse(registeredModel.properties().containsKey(ID_KEY)); + + // Test register model with illegal property + Map illegalProps = ImmutableMap.of("k1", "v1", ID_KEY, "test"); + testPropertyException( + () -> modelOperationDispatcher.registerModel(modelIdent, "comment", illegalProps), + "Properties are reserved and cannot be set", + ID_KEY); + } + + @Test + public void testRegisterAndListModels() { + String schemaName = randomSchemaName(); + NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, schemaName); + schemaOperationDispatcher.createSchema(schemaIdent, "comment", null); + + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + String modelName1 = randomModelName(); + NameIdentifier modelIdent1 = + NameIdentifierUtil.ofModel(metalake, catalog, schemaName, modelName1); + modelOperationDispatcher.registerModel(modelIdent1, "comment", props); + + String modelName2 = randomModelName(); + NameIdentifier modelIdent2 = + NameIdentifierUtil.ofModel(metalake, catalog, schemaName, modelName2); + modelOperationDispatcher.registerModel(modelIdent2, "comment", props); + + NameIdentifier[] modelIdents = modelOperationDispatcher.listModels(modelIdent1.namespace()); + Assertions.assertEquals(2, modelIdents.length); + Set modelIdentSet = Sets.newHashSet(modelIdents); + Assertions.assertTrue(modelIdentSet.contains(modelIdent1)); + Assertions.assertTrue(modelIdentSet.contains(modelIdent2)); + } + + @Test + public void testRegisterAndDeleteModel() { + String schemaName = randomSchemaName(); + NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, schemaName); + schemaOperationDispatcher.createSchema(schemaIdent, "comment", null); + + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + String modelName = randomModelName(); + NameIdentifier modelIdent = + NameIdentifierUtil.ofModel(metalake, catalog, schemaName, modelName); + + modelOperationDispatcher.registerModel(modelIdent, "comment", props); + Assertions.assertTrue(modelOperationDispatcher.deleteModel(modelIdent)); + Assertions.assertFalse(modelOperationDispatcher.deleteModel(modelIdent)); + Assertions.assertThrows( + NoSuchModelException.class, () -> modelOperationDispatcher.getModel(modelIdent)); + + // Test delete in-existent model + Assertions.assertFalse( + modelOperationDispatcher.deleteModel(NameIdentifier.of(metalake, catalog, "inexistent"))); + } + + @Test + public void testLinkAndGetModelVersion() { + String schemaName = randomSchemaName(); + NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, schemaName); + schemaOperationDispatcher.createSchema(schemaIdent, "comment", null); + + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + String modelName = randomModelName(); + NameIdentifier modelIdent = + NameIdentifierUtil.ofModel(metalake, catalog, schemaName, modelName); + + Model model = modelOperationDispatcher.registerModel(modelIdent, "comment", props); + Assertions.assertEquals(0, model.latestVersion()); + + String[] aliases = new String[] {"alias1", "alias2"}; + modelOperationDispatcher.linkModelVersion(modelIdent, "path", aliases, "comment", props); + + ModelVersion linkedModelVersion = modelOperationDispatcher.getModelVersion(modelIdent, 0); + Assertions.assertEquals(0, linkedModelVersion.version()); + Assertions.assertEquals("path", linkedModelVersion.uri()); + Assertions.assertArrayEquals(aliases, linkedModelVersion.aliases()); + Assertions.assertEquals("comment", linkedModelVersion.comment()); + Assertions.assertEquals(props, linkedModelVersion.properties()); + Assertions.assertFalse(linkedModelVersion.properties().containsKey(ID_KEY)); + + // Test get model version with alias + ModelVersion linkedModelVersionWithAlias = + modelOperationDispatcher.getModelVersion(modelIdent, "alias1"); + Assertions.assertEquals(0, linkedModelVersionWithAlias.version()); + Assertions.assertEquals("path", linkedModelVersionWithAlias.uri()); + Assertions.assertArrayEquals(aliases, linkedModelVersionWithAlias.aliases()); + Assertions.assertFalse(linkedModelVersionWithAlias.properties().containsKey(ID_KEY)); + + ModelVersion linkedModelVersionWithAlias2 = + modelOperationDispatcher.getModelVersion(modelIdent, "alias2"); + Assertions.assertEquals(0, linkedModelVersionWithAlias2.version()); + Assertions.assertEquals("path", linkedModelVersionWithAlias2.uri()); + Assertions.assertArrayEquals(aliases, linkedModelVersionWithAlias2.aliases()); + Assertions.assertFalse(linkedModelVersionWithAlias2.properties().containsKey(ID_KEY)); + + // Test Link model version with illegal property + Map illegalProps = ImmutableMap.of("k1", "v1", ID_KEY, "test"); + testPropertyException( + () -> + modelOperationDispatcher.linkModelVersion( + modelIdent, "path", aliases, "comment", illegalProps), + "Properties are reserved and cannot be set", + ID_KEY); + } + + @Test + public void testLinkAndListModelVersion() { + String schemaName = randomSchemaName(); + NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, schemaName); + schemaOperationDispatcher.createSchema(schemaIdent, "comment", null); + + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + String modelName = randomModelName(); + NameIdentifier modelIdent = + NameIdentifierUtil.ofModel(metalake, catalog, schemaName, modelName); + + Model model = modelOperationDispatcher.registerModel(modelIdent, "comment", props); + Assertions.assertEquals(0, model.latestVersion()); + + String[] aliases1 = new String[] {"alias1"}; + String[] aliases2 = new String[] {"alias2"}; + modelOperationDispatcher.linkModelVersion(modelIdent, "path1", aliases1, "comment", props); + modelOperationDispatcher.linkModelVersion(modelIdent, "path2", aliases2, "comment", props); + + int[] versions = modelOperationDispatcher.listModelVersions(modelIdent); + Assertions.assertEquals(2, versions.length); + Set versionSet = Arrays.stream(versions).boxed().collect(Collectors.toSet()); + Assertions.assertTrue(versionSet.contains(0)); + Assertions.assertTrue(versionSet.contains(1)); + } + + @Test + public void testLinkAndDeleteModelVersion() { + String schemaName = randomSchemaName(); + NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, schemaName); + schemaOperationDispatcher.createSchema(schemaIdent, "comment", null); + + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + String modelName = randomModelName(); + NameIdentifier modelIdent = + NameIdentifierUtil.ofModel(metalake, catalog, schemaName, modelName); + + Model model = modelOperationDispatcher.registerModel(modelIdent, "comment", props); + Assertions.assertEquals(0, model.latestVersion()); + + String[] aliases = new String[] {"alias1"}; + modelOperationDispatcher.linkModelVersion(modelIdent, "path", aliases, "comment", props); + Assertions.assertTrue(modelOperationDispatcher.deleteModelVersion(modelIdent, 0)); + Assertions.assertFalse(modelOperationDispatcher.deleteModelVersion(modelIdent, 0)); + Assertions.assertThrows( + NoSuchModelVersionException.class, + () -> modelOperationDispatcher.getModelVersion(modelIdent, 0)); + + // Test delete in-existent model version + Assertions.assertFalse(modelOperationDispatcher.deleteModelVersion(modelIdent, 1)); + + // Tet delete model version with alias + String[] aliases2 = new String[] {"alias2"}; + modelOperationDispatcher.linkModelVersion(modelIdent, "path2", aliases2, "comment", props); + Assertions.assertTrue(modelOperationDispatcher.deleteModelVersion(modelIdent, "alias2")); + Assertions.assertFalse(modelOperationDispatcher.deleteModelVersion(modelIdent, "alias2")); + Assertions.assertThrows( + NoSuchModelVersionException.class, + () -> modelOperationDispatcher.getModelVersion(modelIdent, "alias2")); + } + + private String randomSchemaName() { + return "schema_" + UUID.randomUUID().toString().replace("-", ""); + } + + private String randomModelName() { + return "model_" + UUID.randomUUID().toString().replace("-", ""); + } +} diff --git a/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java b/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java index 4fb98c596b8..f7775ef32e7 100644 --- a/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java +++ b/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java @@ -26,11 +26,13 @@ import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -38,6 +40,8 @@ import org.apache.gravitino.SchemaChange; import org.apache.gravitino.TestColumn; import org.apache.gravitino.TestFileset; +import org.apache.gravitino.TestModel; +import org.apache.gravitino.TestModelVersion; import org.apache.gravitino.TestSchema; import org.apache.gravitino.TestTable; import org.apache.gravitino.TestTopic; @@ -47,8 +51,12 @@ import org.apache.gravitino.exceptions.ConnectionFailedException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.GravitinoRuntimeException; +import org.apache.gravitino.exceptions.ModelAlreadyExistsException; +import org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchFilesetException; +import org.apache.gravitino.exceptions.NoSuchModelException; +import org.apache.gravitino.exceptions.NoSuchModelVersionException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NoSuchTableException; import org.apache.gravitino.exceptions.NoSuchTopicException; @@ -64,6 +72,9 @@ import org.apache.gravitino.messaging.TopicCatalog; import org.apache.gravitino.messaging.TopicChange; import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelCatalog; +import org.apache.gravitino.model.ModelVersion; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.TableCatalog; @@ -76,7 +87,12 @@ import org.slf4j.LoggerFactory; public class TestCatalogOperations - implements CatalogOperations, TableCatalog, FilesetCatalog, TopicCatalog, SupportsSchemas { + implements CatalogOperations, + TableCatalog, + FilesetCatalog, + TopicCatalog, + ModelCatalog, + SupportsSchemas { private static final Logger LOG = LoggerFactory.getLogger(TestCatalogOperations.class); private final Map tables; @@ -87,6 +103,12 @@ public class TestCatalogOperations private final Map topics; + private final Map models; + + private final Map, TestModelVersion> modelVersions; + + private final Map, Integer> modelAliasToVersion; + public static final String FAIL_CREATE = "fail-create"; public static final String FAIL_TEST = "need-fail"; @@ -98,6 +120,9 @@ public TestCatalogOperations(Map config) { schemas = Maps.newHashMap(); filesets = Maps.newHashMap(); topics = Maps.newHashMap(); + models = Maps.newHashMap(); + modelVersions = Maps.newHashMap(); + modelAliasToVersion = Maps.newHashMap(); } @Override @@ -649,6 +674,227 @@ public void testConnection( } } + @Override + public NameIdentifier[] listModels(Namespace namespace) throws NoSuchSchemaException { + NameIdentifier modelSchemaIdent = NameIdentifier.of(namespace.levels()); + if (!schemas.containsKey(modelSchemaIdent)) { + throw new NoSuchSchemaException("Schema %s does not exist", modelSchemaIdent); + } + + return models.keySet().stream() + .filter(ident -> ident.namespace().equals(namespace)) + .toArray(NameIdentifier[]::new); + } + + @Override + public Model getModel(NameIdentifier ident) throws NoSuchModelException { + if (models.containsKey(ident)) { + return models.get(ident); + } else { + throw new NoSuchModelException("Model %s does not exist", ident); + } + } + + @Override + public Model registerModel(NameIdentifier ident, String comment, Map properties) + throws NoSuchSchemaException, ModelAlreadyExistsException { + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + if (!schemas.containsKey(schemaIdent)) { + throw new NoSuchSchemaException("Schema %s does not exist", schemaIdent); + } + + AuditInfo auditInfo = + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); + TestModel model = + TestModel.builder() + .withName(ident.name()) + .withComment(comment) + .withProperties(properties) + .withLatestVersion(0) + .withAuditInfo(auditInfo) + .build(); + + if (models.containsKey(ident)) { + throw new ModelAlreadyExistsException("Model %s already exists", ident); + } else { + models.put(ident, model); + } + + return model; + } + + @Override + public boolean deleteModel(NameIdentifier ident) { + if (!models.containsKey(ident)) { + return false; + } + + models.remove(ident); + + List> deletedVersions = + modelVersions.entrySet().stream() + .filter(e -> e.getKey().getLeft().equals(ident)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + deletedVersions.forEach(modelVersions::remove); + + List> deletedAliases = + modelAliasToVersion.entrySet().stream() + .filter(e -> e.getKey().getLeft().equals(ident)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + deletedAliases.forEach(modelAliasToVersion::remove); + + return true; + } + + @Override + public int[] listModelVersions(NameIdentifier ident) throws NoSuchModelException { + if (!models.containsKey(ident)) { + throw new NoSuchModelException("Model %s does not exist", ident); + } + + return modelVersions.entrySet().stream() + .filter(e -> e.getKey().getLeft().equals(ident)) + .mapToInt(e -> e.getValue().version()) + .toArray(); + } + + @Override + public ModelVersion getModelVersion(NameIdentifier ident, int version) + throws NoSuchModelVersionException { + if (!models.containsKey(ident)) { + throw new NoSuchModelVersionException("Model %s does not exist", ident); + } + + Pair versionPair = Pair.of(ident, version); + if (!modelVersions.containsKey(versionPair)) { + throw new NoSuchModelVersionException("Model version %s does not exist", versionPair); + } + + return modelVersions.get(versionPair); + } + + @Override + public ModelVersion getModelVersion(NameIdentifier ident, String alias) + throws NoSuchModelVersionException { + if (!models.containsKey(ident)) { + throw new NoSuchModelVersionException("Model %s does not exist", ident); + } + + Pair aliasPair = Pair.of(ident, alias); + if (!modelAliasToVersion.containsKey(aliasPair)) { + throw new NoSuchModelVersionException("Model version %s does not exist", alias); + } + + int version = modelAliasToVersion.get(aliasPair); + Pair versionPair = Pair.of(ident, version); + if (!modelVersions.containsKey(versionPair)) { + throw new NoSuchModelVersionException("Model version %s does not exist", versionPair); + } + + return modelVersions.get(versionPair); + } + + @Override + public void linkModelVersion( + NameIdentifier ident, + String uri, + String[] aliases, + String comment, + Map properties) + throws NoSuchModelException, ModelVersionAliasesAlreadyExistException { + if (!models.containsKey(ident)) { + throw new NoSuchModelException("Model %s does not exist", ident); + } + + String[] aliasArray = aliases != null ? aliases : new String[0]; + for (String alias : aliasArray) { + Pair aliasPair = Pair.of(ident, alias); + if (modelAliasToVersion.containsKey(aliasPair)) { + throw new ModelVersionAliasesAlreadyExistException( + "Model version alias %s already exists", alias); + } + } + + int version = models.get(ident).latestVersion(); + TestModelVersion modelVersion = + TestModelVersion.builder() + .withVersion(version) + .withAliases(aliases) + .withComment(comment) + .withUri(uri) + .withProperties(properties) + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); + Pair versionPair = Pair.of(ident, version); + modelVersions.put(versionPair, modelVersion); + for (String alias : aliasArray) { + Pair aliasPair = Pair.of(ident, alias); + modelAliasToVersion.put(aliasPair, version); + } + + TestModel model = models.get(ident); + TestModel updatedModel = + TestModel.builder() + .withName(model.name()) + .withComment(model.comment()) + .withProperties(model.properties()) + .withLatestVersion(version + 1) + .withAuditInfo(model.auditInfo()) + .build(); + models.put(ident, updatedModel); + } + + @Override + public boolean deleteModelVersion(NameIdentifier ident, int version) { + if (!models.containsKey(ident)) { + return false; + } + + Pair versionPair = Pair.of(ident, version); + if (!modelVersions.containsKey(versionPair)) { + return false; + } + + TestModelVersion modelVersion = modelVersions.remove(versionPair); + if (modelVersion.aliases() != null) { + for (String alias : modelVersion.aliases()) { + Pair aliasPair = Pair.of(ident, alias); + modelAliasToVersion.remove(aliasPair); + } + } + + return true; + } + + @Override + public boolean deleteModelVersion(NameIdentifier ident, String alias) { + if (!models.containsKey(ident)) { + return false; + } + + Pair aliasPair = Pair.of(ident, alias); + if (!modelAliasToVersion.containsKey(aliasPair)) { + return false; + } + + int version = modelAliasToVersion.remove(aliasPair); + Pair versionPair = Pair.of(ident, version); + if (!modelVersions.containsKey(versionPair)) { + return false; + } + + TestModelVersion modelVersion = modelVersions.remove(versionPair); + for (String modelVersionAlias : modelVersion.aliases()) { + Pair modelAliasPair = Pair.of(ident, modelVersionAlias); + modelAliasToVersion.remove(modelAliasPair); + } + + return true; + } + private boolean hasCallerContext() { return CallerContext.CallerContextHolder.get() != null && CallerContext.CallerContextHolder.get().context() != null From b2485bb695894a75e3cf80d4eb69ba303ae8cbc3 Mon Sep 17 00:00:00 2001 From: Cheng-Yi Shih <48374270+cool9850311@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:44:40 +0800 Subject: [PATCH 16/47] [#5911] fix(docs): Fix the wrong possible values. (#5922) ### What changes were proposed in this pull request? Fix the wrong possible values. Remove "COLUMN" from metadataObjectType possible values in get/set owner. ### Why are the changes needed? Fix: #5911 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Just documents. --- docs/open-api/openapi.yaml | 3 +-- docs/open-api/owners.yaml | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/open-api/openapi.yaml b/docs/open-api/openapi.yaml index 0985a60eddb..dd0564a7f9c 100644 --- a/docs/open-api/openapi.yaml +++ b/docs/open-api/openapi.yaml @@ -469,6 +469,7 @@ components: schema: type: string enum: + - "METALAKE" - "CATALOG" - "SCHEMA" - "TABLE" @@ -476,8 +477,6 @@ components: - "FILESET" - "TOPIC" - "ROLE" - - "METALAKE" - metadataObjectFullName: name: metadataObjectFullName in: path diff --git a/docs/open-api/owners.yaml b/docs/open-api/owners.yaml index c0c6b8173f3..0ef0d4e9f01 100644 --- a/docs/open-api/owners.yaml +++ b/docs/open-api/owners.yaml @@ -22,7 +22,7 @@ paths: /metalakes/{metalake}/owners/{metadataObjectType}/{metadataObjectFullName}: parameters: - $ref: "./openapi.yaml#/components/parameters/metalake" - - $ref: "./openapi.yaml#/components/parameters/metadataObjectType" + - $ref: "#/components/parameters/metadataObjectTypeOfOwner" - $ref: "./openapi.yaml#/components/parameters/metadataObjectFullName" put: @@ -171,4 +171,21 @@ components: "org.apache.gravitino.exceptions.NotFoundException: Metadata object or owner does not exist", "..." ] - } \ No newline at end of file + } + + parameters: + metadataObjectTypeOfOwner: + name: metadataObjectType + in: path + description: The type of the metadata object + required: true + schema: + type: string + enum: + - "METALAKE" + - "CATALOG" + - "SCHEMA" + - "TABLE" + - "FILESET" + - "TOPIC" + - "ROLE" \ No newline at end of file From 663f25ecb16151f86346bc64c9a70160c2b40204 Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:53:55 +0800 Subject: [PATCH 17/47] [#5827] improvement(CLI): Fix CLI throws an obscure error when Delete a table with a missing table name (#5906) ### What changes were proposed in this pull request? Fix CLI throws an obscure error when Delete a table with a missing table name.it should give clearer hints. ### Why are the changes needed? Fix: #5827 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? #### local bash test ```bash bin/gcli.sh table delete --metalake demo_metalake Missing required argument(s): catalog, schema, table bin/gcli.sh table delete --metalake demo_metalake --name Hive_catalog # output: Missing required argument(s): schema, table bin/gcli.sh table delete --metalake demo_metalake --name Hive_catalog.default # output: Missing required argument(s): table bin/gcli.sh table details --metalake demo_metalake --name Hive_catalog.default # output # Malformed entity name. # Missing required argument(s): table ``` #### Unit test add some test case to test whether the command fuse is executed correctly. --- .../gravitino/cli/GravitinoCommandLine.java | 22 ++- .../gravitino/cli/TestTableCommands.java | 147 ++++++++++++++++++ 2 files changed, 161 insertions(+), 8 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index a27e3cb2a08..7be65830a19 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -381,27 +381,33 @@ private void handleTableCommand() { String schema = name.getSchemaName(); Command.setAuthenticationMode(auth, userName); + List missingEntities = + Stream.of( + catalog == null ? CommandEntities.CATALOG : null, + schema == null ? CommandEntities.SCHEMA : null) + .filter(Objects::nonNull) + .collect(Collectors.toList()); // Handle CommandActions.LIST action separately as it doesn't require the `table` if (CommandActions.LIST.equals(command)) { - List missingEntities = - Stream.of( - metalake == null ? CommandEntities.METALAKE : null, - catalog == null ? CommandEntities.CATALOG : null, - schema == null ? CommandEntities.SCHEMA : null) - .filter(Objects::nonNull) - .collect(Collectors.toList()); if (!missingEntities.isEmpty()) { System.err.println( "Missing required argument(s): " + Joiner.on(", ").join(missingEntities)); Main.exit(-1); } - newListTables(url, ignore, metalake, catalog, schema).handle(); return; } String table = name.getTableName(); + if (table == null) { + missingEntities.add(CommandEntities.TABLE); + } + + if (!missingEntities.isEmpty()) { + System.err.println("Missing required argument(s): " + Joiner.on(", ").join(missingEntities)); + Main.exit(-1); + } switch (command) { case CommandActions.DETAILS: diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java index 07cbdbdcc6c..32c289cfd85 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java @@ -19,12 +19,17 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateTable; @@ -41,6 +46,7 @@ import org.apache.gravitino.cli.commands.TableSortOrder; import org.apache.gravitino.cli.commands.UpdateTableComment; import org.apache.gravitino.cli.commands.UpdateTableName; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,10 +54,28 @@ class TestTableCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -410,4 +434,127 @@ void testCreateTable() { commandLine.handleCommandLine(); verify(mockCreate).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testListTableWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TABLE, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListTables(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null); + assertTrue( + errContent + .toString() + .contains( + "Missing required argument(s): " + + CommandEntities.CATALOG + + ", " + + CommandEntities.SCHEMA)); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListTableWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TABLE, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListTables(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null); + assertTrue( + errContent.toString().contains("Missing required argument(s): " + CommandEntities.SCHEMA)); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDetailTableWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TABLE, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTableDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null); + assertTrue( + errContent + .toString() + .contains( + "Missing required argument(s): " + + CommandEntities.CATALOG + + ", " + + CommandEntities.SCHEMA + + ", " + + CommandEntities.TABLE)); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDetailTableWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TABLE, CommandActions.DETAILS)); + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTableDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null); + assertTrue( + errContent + .toString() + .contains( + "Missing required argument(s): " + + CommandEntities.SCHEMA + + ", " + + CommandEntities.TABLE)); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDetailTableWithoutTable() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TABLE, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTableDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + assertTrue( + errContent.toString().contains("Missing required argument(s): " + CommandEntities.TABLE)); + } } From 4cce13c770bb0fa7a8ce125ef2c8ee8c837de7f8 Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Mon, 23 Dec 2024 17:03:24 +0800 Subject: [PATCH 18/47] [#5926] fix(CLI): Fix Missleading error message in Gravitino CLI when missing schema (#5939) ### What changes were proposed in this pull request? Fix schema arguments validation just like table commands. running command like `gcli schema details --metalake metalake_demo --name catalog_postgres -I` give Missleading error message as below. ```bash Malformed entity name. Invalid string to encode: null ``` It should display a friendly error message. ### Why are the changes needed? Fix: #5926 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? ```bash bin/gcli.sh schema details --m demo_metalake -i # output # Missing --name option. # Missing --name option. # Missing required argument(s): catalog, schema bin/gcli.sh schema details --m demo_metalake --name Hive_catalog -i # output # Malformed entity name. # Missing required argument(s): schema bin/gcli.sh schema details --m demo_metalake --name Hive_catalog.default -i # ouput: default,Default Hive database bin/gcli.sh schema list --m demo_metalake -i # output: # Missing --name option. # Missing required argument(s): catalog bin/gcli.sh schema list --m demo_metalake -i --name Hive_catalog # correct output ``` --- .../gravitino/cli/GravitinoCommandLine.java | 16 ++++ .../gravitino/cli/TestSchemaCommands.java | 87 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 7be65830a19..16261a7b4af 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -21,6 +21,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -318,14 +319,29 @@ private void handleSchemaCommand() { String catalog = name.getCatalogName(); Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); + if (metalake == null) missingEntities.add(CommandEntities.METALAKE); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); // Handle the CommandActions.LIST action separately as it doesn't use `schema` if (CommandActions.LIST.equals(command)) { + if (!missingEntities.isEmpty()) { + System.err.println("Missing required argument(s): " + COMMA_JOINER.join(missingEntities)); + Main.exit(-1); + } newListSchema(url, ignore, metalake, catalog).handle(); return; } String schema = name.getSchemaName(); + if (schema == null) { + missingEntities.add(CommandEntities.SCHEMA); + } + + if (!missingEntities.isEmpty()) { + System.err.println("Missing required argument(s): " + COMMA_JOINER.join(missingEntities)); + Main.exit(-1); + } switch (command) { case CommandActions.DETAILS: diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestSchemaCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestSchemaCommands.java index 89cc72bcdbf..190e866355b 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestSchemaCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestSchemaCommands.java @@ -19,12 +19,17 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateSchema; @@ -35,6 +40,7 @@ import org.apache.gravitino.cli.commands.SchemaAudit; import org.apache.gravitino.cli.commands.SchemaDetails; import org.apache.gravitino.cli.commands.SetSchemaProperty; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -42,10 +48,28 @@ class TestSchemaCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -245,4 +269,67 @@ void testListSchemaPropertiesCommand() { commandLine.handleCommandLine(); verify(mockListProperties).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testListSchemaWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.SCHEMA, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListSchema(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null); + assertTrue( + errContent.toString().contains("Missing required argument(s): " + CommandEntities.CATALOG)); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDetailsSchemaWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.SCHEMA, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + assertTrue( + errContent + .toString() + .contains( + "Missing required argument(s): " + + CommandEntities.CATALOG + + ", " + + CommandEntities.SCHEMA)); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDetailsSchemaWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.SCHEMA, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + assertTrue( + errContent.toString().contains("Missing required argument(s): " + CommandEntities.SCHEMA)); + } } From 624839564ff7161556920955f04e66e86fbd3201 Mon Sep 17 00:00:00 2001 From: roryqi Date: Mon, 23 Dec 2024 17:03:45 +0800 Subject: [PATCH 19/47] [#5947] fix(auth): It will throw error if we enable authorization and rename catalog (#5949) ### What changes were proposed in this pull request? Fix the issue of renaming catalogs or metalakes. ### Why are the changes needed? Fix: #5947 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add UT. --- .../ranger/integration/test/RangerBaseE2EIT.java | 16 ++++++++++++++++ .../authorization/AuthorizationUtils.java | 9 +++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java index c7c9ec02f22..1fb9677d528 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java @@ -29,8 +29,10 @@ import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.gravitino.Catalog; +import org.apache.gravitino.CatalogChange; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.auth.AuthConstants; import org.apache.gravitino.authorization.Owner; @@ -203,6 +205,20 @@ protected static void waitForUpdatingPolicies() { protected abstract void testAlterTable(); + // ISSUE-5947: can't rename a catalog or a metalake + @Test + void testRenameMetalakeOrCatalog() { + Assertions.assertDoesNotThrow( + () -> client.alterMetalake(metalakeName, MetalakeChange.rename("new_name"))); + Assertions.assertDoesNotThrow( + () -> client.alterMetalake("new_name", MetalakeChange.rename(metalakeName))); + + Assertions.assertDoesNotThrow( + () -> metalake.alterCatalog(catalogName, CatalogChange.rename("new_name"))); + Assertions.assertDoesNotThrow( + () -> metalake.alterCatalog("new_name", CatalogChange.rename(catalogName))); + } + @Test protected void testCreateSchema() throws InterruptedException { // Choose a catalog diff --git a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java index 61aa86f425e..0e236b72635 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java @@ -274,9 +274,14 @@ public static void authorizationPluginRenamePrivileges( NameIdentifierUtil.toMetadataObject(NameIdentifier.of(ident.namespace(), newName), type); MetadataObjectChange renameObject = MetadataObjectChange.rename(oldMetadataObject, newMetadataObject); + + String metalake = type == Entity.EntityType.METALAKE ? newName : ident.namespace().level(0); + + // For a renamed catalog, we should pass the new name catalog, otherwise we can't find the + // catalog in the entity store callAuthorizationPluginForMetadataObject( - ident.namespace().level(0), - oldMetadataObject, + metalake, + newMetadataObject, authorizationPlugin -> { authorizationPlugin.onMetadataUpdated(renameObject); }); From dfd490a206a4c083d87eac6018c0020a707f412d Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Mon, 23 Dec 2024 17:07:33 +0800 Subject: [PATCH 20/47] [#5928] fix(CLI): Fix columns details command produces no output (#5945) ### What changes were proposed in this pull request? Fix columns details command produces no output, when command without --audit option, cli should tell user that command is not support. ### Why are the changes needed? Fix: #5928 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? local test. --- .../gravitino/cli/GravitinoCommandLine.java | 3 ++ .../gravitino/cli/TestColumnCommands.java | 47 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 16261a7b4af..f1f469aa430 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -799,6 +799,9 @@ private void handleColumnCommand() { case CommandActions.DETAILS: if (line.hasOption(GravitinoOptions.AUDIT)) { newColumnAudit(url, ignore, metalake, catalog, schema, table, column).handle(); + } else { + System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); } break; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java index e26759e2d4c..2eb4c536480 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java @@ -19,12 +19,18 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.AddColumn; @@ -38,17 +44,30 @@ import org.apache.gravitino.cli.commands.UpdateColumnName; import org.apache.gravitino.cli.commands.UpdateColumnNullability; import org.apache.gravitino.cli.commands.UpdateColumnPosition; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestColumnCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -98,6 +117,34 @@ void testColumnAuditCommand() { verify(mockAudit).handle(); } + @Test + void testColumnDetailsCommand() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)) + .thenReturn("catalog.schema.users.name"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newColumnAudit( + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + "catalog", + "schema", + "users", + "name"); + + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.UNSUPPORTED_ACTION); + } + @Test void testAddColumn() { AddColumn mockAddColumn = mock(AddColumn.class); From 77bd191b15c15b274ef97d8f9090f78a19143c0a Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Mon, 23 Dec 2024 17:08:54 +0800 Subject: [PATCH 21/47] [#5929] improvement(CLI): fix cli details command produce no output (#5941) ### What changes were proposed in this pull request? If a user has no roles then no output is produced from `user\group details` command, it should give some help information. ### Why are the changes needed? Fix: #5929 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? ```bash bin/gcli.sh user details -m demo_metalake --user test_user # output: User has no roles. bin/gcli.sh group details -m demo_metalake --group group_no_role # output: Groups has no roles. ``` --------- Co-authored-by: Qiming Teng --- .../java/org/apache/gravitino/cli/commands/GroupDetails.java | 2 +- .../java/org/apache/gravitino/cli/commands/UserDetails.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java index 4df87b5fa8e..7217d5ad3bd 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java @@ -60,7 +60,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = String.join(",", roles); + String all = roles.isEmpty() ? "The group has no roles." : String.join(",", roles); System.out.println(all.toString()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java index 1d59c83e529..e37f8e6f139 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java @@ -60,7 +60,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = String.join(",", roles); + String all = roles.isEmpty() ? "The user has no roles." : String.join(",", roles); System.out.println(all.toString()); } From 727ef88883b25285bbb68f3288bb3f4f16fefbd9 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 23 Dec 2024 21:14:10 +0800 Subject: [PATCH 22/47] [MINOR] Remove add-to-project Github Action (#5951) ### What changes were proposed in this pull request? Remove the `add-to-project` action as the token is expired and we have no permission to update the token as a ASF project. ### Why are the changes needed? Fix the Github Action failure. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? CI. --- .github/workflows/add-to-project.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .github/workflows/add-to-project.yml diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml deleted file mode 100644 index 6ac8c758034..00000000000 --- a/.github/workflows/add-to-project.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Add issue to project - -on: - issues: - types: - - opened - -jobs: - add-to-project: - name: Add issue to project - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@v0.5.0 - with: - project-url: https://github.com/orgs/datastrato/projects/1 - github-token: ${{ secrets.ADD_ISSUE_TO_PROJECT }} From df97ea98a5125fe8b353faafa2bcf7064c99e897 Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Tue, 24 Dec 2024 05:31:52 +0800 Subject: [PATCH 23/47] [#5924] improvement(CLI): fix cli list command produce no outputs (#5942) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this pull request? When using the list command, it’s helpful to provide a friendly information if no data is retrieve. ### Why are the changes needed? Fix: #5924 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? ```bash bin/gcli.sh catalog list -m demo_metalake2 # No catalogs exists. bin/gcli.sh table list -m demo_metalake --name Hive_catalog.empty # output: No tables exists. ``` --------- Co-authored-by: Qiming Teng --- .../java/org/apache/gravitino/cli/commands/ListAllTags.java | 2 +- .../org/apache/gravitino/cli/commands/ListCatalogs.java | 6 +++++- .../org/apache/gravitino/cli/commands/ListFilesets.java | 2 +- .../java/org/apache/gravitino/cli/commands/ListGroups.java | 2 +- .../org/apache/gravitino/cli/commands/ListMetalakes.java | 6 +++++- .../java/org/apache/gravitino/cli/commands/ListRoles.java | 2 +- .../java/org/apache/gravitino/cli/commands/ListSchema.java | 2 +- .../java/org/apache/gravitino/cli/commands/ListTables.java | 5 ++++- .../java/org/apache/gravitino/cli/commands/ListTopics.java | 5 ++++- 9 files changed, 23 insertions(+), 9 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java index fa6c74c7afa..cded12808d9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java @@ -53,7 +53,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = String.join(",", tags); + String all = tags.length == 0 ? "No tags exist." : String.join(",", tags); System.out.println(all.toString()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java index e6aaf811ec9..eb9c960b14e 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java @@ -49,7 +49,11 @@ public void handle() { try { GravitinoClient client = buildClient(metalake); catalogs = client.listCatalogsInfo(); - output(catalogs); + if (catalogs.length == 0) { + System.out.println("No catalogs exist."); + } else { + output(catalogs); + } } catch (NoSuchMetalakeException err) { exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java index 34839f683c5..d00ba3e6ba5 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java @@ -71,7 +71,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = Joiner.on(",").join(filesets); + String all = filesets.length == 0 ? "No filesets exist." : Joiner.on(",").join(filesets); System.out.println(all.toString()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java index fd9009a755a..a517b4daed8 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java @@ -53,7 +53,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = String.join(",", groups); + String all = groups.length == 0 ? "No groups exist." : String.join(",", groups); System.out.println(all.toString()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java index ee5ac81d646..b2388e5cd3d 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java @@ -43,7 +43,11 @@ public void handle() { try { GravitinoAdminClient client = buildAdminClient(); metalakes = client.listMetalakes(); - output(metalakes); + if (metalakes.length == 0) { + System.out.println("No metalakes exist."); + } else { + output(metalakes); + } } catch (Exception exp) { exitWithError(exp.getMessage()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java index a7bb1cd20f7..2ecb35bd093 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java @@ -53,7 +53,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = String.join(",", roles); + String all = roles.length == 0 ? "No roles exist." : String.join(",", roles); System.out.println(all.toString()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java index cf5fe487cc8..110a6477a62 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java @@ -60,7 +60,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = Joiner.on(",").join(schemas); + String all = schemas.length == 0 ? "No schemas exist." : Joiner.on(",").join(schemas); System.out.println(all.toString()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java index e6afb9b51c0..41a71e87c00 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java @@ -61,7 +61,10 @@ public void handle() { tableNames.add(tables[i].name()); } - String all = Joiner.on(System.lineSeparator()).join(tableNames); + String all = + tableNames.isEmpty() + ? "No tables exist." + : Joiner.on(System.lineSeparator()).join(tableNames); System.out.println(all.toString()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopics.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopics.java index af4cc217713..a2da6a69ad7 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopics.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTopics.java @@ -66,7 +66,10 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = Joiner.on(",").join(Arrays.stream(topics).map(topic -> topic.name()).iterator()); + String all = + topics.length == 0 + ? "No topics exist." + : Joiner.on(",").join(Arrays.stream(topics).map(topic -> topic.name()).iterator()); System.out.println(all); } } From 24f8c457ee0a2358983c9892204b8f2f05fc960c Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Tue, 24 Dec 2024 05:34:39 +0800 Subject: [PATCH 24/47] [#5826] fix(CLI): Fix Dropping a metalake via the Gravitino CLI gives an "in use" exception (#5907) ### What changes were proposed in this pull request? When attempting to drop a metalake via the Gravitino CLI, an "in use" exception is raised. If the metalake is currently in use, the system should provide additional hints or details to help the user identify the issue. Additionally, it would be beneficial to add enable and disable commands to the client. These commands would allow users to enable or disable a catalog or metalake directly through the client interface. #### enable command - `metalake update -m demo_metalake --enable` : enable a metalake - `metalake update -m demo_metalake --enable --all`: enable a metalake and all catalogs in this metalake. - `catalog update -m demo_metalake --name Hive_catalog --enable`: enable a catalog - `catalog update -m demo_metalake --name Hive_catalog --enable --all`: enable a catalog and it's metalake #### disable command - `metalake update -m demo_metalake --disable `: disable a metalake - `catalog update -m demo_metalake --name Hive_catalog --disable `: disable a catalog ### Why are the changes needed? Fix: #5826 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? ```bash bin/gcli.sh metalake delete --metalake demo_metalake2 # output: # demo_metalake2 in use, please use disable command disable it first. # demo_metalake2 not deleted. bin/gcli.sh metalake update --metalake demo_metalake --enable # demo_metalake has been enabled. bin/gcli.sh metalake update --metalake demo_metalake --all --enable # demo_metalake has been enabled. and all catalogs in this metalake have been enabled. bin/gcli.sh catalog update --metalake demo_metalake --name Hive_catalog --enable # demo_metalake.Hive_catalog has been enabled. bin/gcli.sh catalog update --metalake demo_metalake --name Hive_catalog --enable --all # demo_metalake.Hive_catalog has been enabled. bin/gcli.sh metalake update --metalake demo_metalake --disable # demo_metalake has been disabled. bin/gcli.sh catalog update --metalake demo_metalake --name Hive_catalog --disable # demo_metalake.Hive_catalog has been disabled. bin/gcli.sh catalog update --metalake demo_metalake --name Hive_catalog --disable --enable # Unable to enable and disable at the same time ``` --------- Co-authored-by: Qiming Teng --- .../gravitino/cli/GravitinoCommandLine.java | 24 ++++ .../gravitino/cli/GravitinoOptions.java | 6 + .../gravitino/cli/TestableCommandLine.java | 23 ++++ .../cli/commands/CatalogDisable.java | 62 ++++++++++ .../gravitino/cli/commands/CatalogEnable.java | 74 ++++++++++++ .../gravitino/cli/commands/DeleteCatalog.java | 3 + .../cli/commands/DeleteMetalake.java | 3 + .../cli/commands/MetalakeDisable.java | 56 +++++++++ .../cli/commands/MetalakeEnable.java | 72 +++++++++++ .../cli/src/main/resources/catalog_help.txt | 8 +- .../cli/src/main/resources/metalake_help.txt | 6 + .../gravitino/cli/TestCatalogCommands.java | 114 ++++++++++++++++++ .../org/apache/gravitino/cli/TestMain.java | 1 + .../gravitino/cli/TestMetalakeCommands.java | 101 ++++++++++++++++ docs/cli.md | 36 ++++++ 15 files changed, 588 insertions(+), 1 deletion(-) create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogDisable.java create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogEnable.java create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeDisable.java create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeEnable.java diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index f1f469aa430..3f37e84ff65 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -213,6 +213,18 @@ private void handleMetalakeCommand() { break; case CommandActions.UPDATE: + if (line.hasOption(GravitinoOptions.ENABLE) && line.hasOption(GravitinoOptions.DISABLE)) { + System.err.println("Unable to enable and disable at the same time"); + Main.exit(-1); + } + if (line.hasOption(GravitinoOptions.ENABLE)) { + boolean enableAllCatalogs = line.hasOption(GravitinoOptions.ALL); + newMetalakeEnable(url, ignore, metalake, enableAllCatalogs).handle(); + } + if (line.hasOption(GravitinoOptions.DISABLE)) { + newMetalakeDisable(url, ignore, metalake).handle(); + } + if (line.hasOption(GravitinoOptions.COMMENT)) { comment = line.getOptionValue(GravitinoOptions.COMMENT); newUpdateMetalakeComment(url, ignore, metalake, comment).handle(); @@ -290,6 +302,18 @@ private void handleCatalogCommand() { break; case CommandActions.UPDATE: + if (line.hasOption(GravitinoOptions.ENABLE) && line.hasOption(GravitinoOptions.DISABLE)) { + System.err.println("Unable to enable and disable at the same time"); + Main.exit(-1); + } + if (line.hasOption(GravitinoOptions.ENABLE)) { + boolean enableMetalake = line.hasOption(GravitinoOptions.ALL); + newCatalogEnable(url, ignore, metalake, catalog, enableMetalake).handle(); + } + if (line.hasOption(GravitinoOptions.DISABLE)) { + newCatalogDisable(url, ignore, metalake, catalog).handle(); + } + if (line.hasOption(GravitinoOptions.COMMENT)) { String updateComment = line.getOptionValue(GravitinoOptions.COMMENT); newUpdateCatalogComment(url, ignore, metalake, catalog, updateComment).handle(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java index a42591026a6..657566036dc 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java @@ -59,6 +59,9 @@ public class GravitinoOptions { public static final String USER = "user"; public static final String VALUE = "value"; public static final String VERSION = "version"; + public static final String ALL = "all"; + public static final String ENABLE = "enable"; + public static final String DISABLE = "disable"; /** * Builds and returns the CLI options for Gravitino. @@ -84,6 +87,8 @@ public Options options() { options.addOption(createSimpleOption(PARTITION, "display partition information")); options.addOption(createSimpleOption("o", OWNER, "display entity owner")); options.addOption(createSimpleOption(null, SORTORDER, "display sortorder information")); + options.addOption(createSimpleOption(null, ENABLE, "enable entities")); + options.addOption(createSimpleOption(null, DISABLE, "disable entities")); // Create/update options options.addOption(createArgOption(RENAME, "new entity name")); @@ -102,6 +107,7 @@ public Options options() { options.addOption(createArgOption(DEFAULT, "default column value")); options.addOption(createSimpleOption("o", OWNER, "display entity owner")); options.addOption(createArgOption(COLUMNFILE, "CSV file describing columns")); + options.addOption(createSimpleOption(null, ALL, "all operation for --enable")); // Options that support multiple values options.addOption(createArgsOption("p", PROPERTIES, "property name/value pairs")); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java index 41909f7209e..effe0da1f10 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java @@ -26,6 +26,8 @@ import org.apache.gravitino.cli.commands.AddRoleToUser; import org.apache.gravitino.cli.commands.CatalogAudit; import org.apache.gravitino.cli.commands.CatalogDetails; +import org.apache.gravitino.cli.commands.CatalogDisable; +import org.apache.gravitino.cli.commands.CatalogEnable; import org.apache.gravitino.cli.commands.ClientVersion; import org.apache.gravitino.cli.commands.ColumnAudit; import org.apache.gravitino.cli.commands.CreateCatalog; @@ -75,6 +77,8 @@ import org.apache.gravitino.cli.commands.ListUsers; import org.apache.gravitino.cli.commands.MetalakeAudit; import org.apache.gravitino.cli.commands.MetalakeDetails; +import org.apache.gravitino.cli.commands.MetalakeDisable; +import org.apache.gravitino.cli.commands.MetalakeEnable; import org.apache.gravitino.cli.commands.OwnerDetails; import org.apache.gravitino.cli.commands.RemoveAllTags; import org.apache.gravitino.cli.commands.RemoveCatalogProperty; @@ -884,4 +888,23 @@ protected RevokePrivilegesFromRole newRevokePrivilegesFromRole( String[] privileges) { return new RevokePrivilegesFromRole(url, ignore, metalake, role, entity, privileges); } + + protected MetalakeEnable newMetalakeEnable( + String url, boolean ignore, String metalake, boolean enableAllCatalogs) { + return new MetalakeEnable(url, ignore, metalake, enableAllCatalogs); + } + + protected MetalakeDisable newMetalakeDisable(String url, boolean ignore, String metalake) { + return new MetalakeDisable(url, ignore, metalake); + } + + protected CatalogEnable newCatalogEnable( + String url, boolean ignore, String metalake, String catalog, boolean enableMetalake) { + return new CatalogEnable(url, ignore, metalake, catalog, enableMetalake); + } + + protected CatalogDisable newCatalogDisable( + String url, boolean ignore, String metalake, String catalog) { + return new CatalogDisable(url, ignore, metalake, catalog); + } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogDisable.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogDisable.java new file mode 100644 index 00000000000..620a4291eea --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogDisable.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.cli.commands; + +import org.apache.gravitino.cli.ErrorMessages; +import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.exceptions.NoSuchCatalogException; +import org.apache.gravitino.exceptions.NoSuchMetalakeException; + +/** Disable catalog. */ +public class CatalogDisable extends Command { + + private final String metalake; + private final String catalog; + + /** + * Disable catalog + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + * @param metalake The name of the metalake. + * @param catalog The name of the catalog. + */ + public CatalogDisable(String url, boolean ignoreVersions, String metalake, String catalog) { + super(url, ignoreVersions); + this.metalake = metalake; + this.catalog = catalog; + } + + /** Disable catalog. */ + @Override + public void handle() { + try { + GravitinoClient client = buildClient(metalake); + client.disableCatalog(catalog); + } catch (NoSuchMetalakeException noSuchMetalakeException) { + exitWithError(ErrorMessages.UNKNOWN_METALAKE); + } catch (NoSuchCatalogException noSuchCatalogException) { + exitWithError(ErrorMessages.UNKNOWN_CATALOG); + } catch (Exception exp) { + exitWithError(exp.getMessage()); + } + + System.out.println(metalake + "." + catalog + " has been disabled."); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogEnable.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogEnable.java new file mode 100644 index 00000000000..8646baee292 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CatalogEnable.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.cli.commands; + +import org.apache.gravitino.cli.ErrorMessages; +import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; +import org.apache.gravitino.exceptions.NoSuchCatalogException; +import org.apache.gravitino.exceptions.NoSuchMetalakeException; + +/** Enable catalog. */ +public class CatalogEnable extends Command { + private final String metalake; + private final String catalog; + private final boolean enableMetalake; + + /** + * Enable catalog + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + * @param metalake The name of the metalake. + * @param catalog The name of the catalog. + * @param enableMetalake Whether to enable it's metalake + */ + public CatalogEnable( + String url, boolean ignoreVersions, String metalake, String catalog, boolean enableMetalake) { + super(url, ignoreVersions); + this.metalake = metalake; + this.catalog = catalog; + this.enableMetalake = enableMetalake; + } + + /** Enable catalog. */ + @Override + public void handle() { + try { + if (enableMetalake) { + GravitinoAdminClient adminClient = buildAdminClient(); + adminClient.enableMetalake(metalake); + } + GravitinoClient client = buildClient(metalake); + client.enableCatalog(catalog); + } catch (NoSuchMetalakeException noSuchMetalakeException) { + exitWithError(ErrorMessages.UNKNOWN_METALAKE); + } catch (NoSuchCatalogException noSuchCatalogException) { + exitWithError(ErrorMessages.UNKNOWN_CATALOG); + } catch (MetalakeNotInUseException notInUseException) { + exitWithError( + metalake + " not in use. please use --recursive option, or enable metalake first"); + } catch (Exception exp) { + exitWithError(exp.getMessage()); + } + + System.out.println(metalake + "." + catalog + " has been enabled."); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteCatalog.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteCatalog.java index 6c5fbaee97d..6aa8e5ad904 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteCatalog.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteCatalog.java @@ -22,6 +22,7 @@ import org.apache.gravitino.cli.AreYouSure; import org.apache.gravitino.cli.ErrorMessages; import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; @@ -64,6 +65,8 @@ public void handle() { exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchCatalogException err) { exitWithError(ErrorMessages.UNKNOWN_CATALOG); + } catch (CatalogInUseException catalogInUseException) { + System.err.println(catalog + " in use, please disable it first."); } catch (Exception exp) { exitWithError(exp.getMessage()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteMetalake.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteMetalake.java index 386dde92130..e88ae41486f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteMetalake.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteMetalake.java @@ -22,6 +22,7 @@ import org.apache.gravitino.cli.AreYouSure; import org.apache.gravitino.cli.ErrorMessages; import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; public class DeleteMetalake extends Command { @@ -56,6 +57,8 @@ public void handle() { deleted = client.dropMetalake(metalake); } catch (NoSuchMetalakeException err) { exitWithError(ErrorMessages.UNKNOWN_METALAKE); + } catch (MetalakeInUseException inUseException) { + System.err.println(metalake + " in use, please disable it first."); } catch (Exception exp) { exitWithError(exp.getMessage()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeDisable.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeDisable.java new file mode 100644 index 00000000000..02e33a45d45 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeDisable.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli.commands; + +import org.apache.gravitino.cli.ErrorMessages; +import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.exceptions.NoSuchMetalakeException; + +/** Disable metalake. */ +public class MetalakeDisable extends Command { + private String metalake; + + /** + * Disable metalake + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + * @param metalake The name of the metalake. + */ + public MetalakeDisable(String url, boolean ignoreVersions, String metalake) { + super(url, ignoreVersions); + this.metalake = metalake; + } + + /** Disable metalake. */ + @Override + public void handle() { + try { + GravitinoAdminClient client = buildAdminClient(); + client.disableMetalake(metalake); + } catch (NoSuchMetalakeException err) { + exitWithError(ErrorMessages.UNKNOWN_METALAKE); + } catch (Exception exp) { + exitWithError(exp.getMessage()); + } + + System.out.println(metalake + " has been disabled."); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeEnable.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeEnable.java new file mode 100644 index 00000000000..34ba23a61bb --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/MetalakeEnable.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli.commands; + +import java.util.Arrays; +import org.apache.gravitino.cli.ErrorMessages; +import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.exceptions.NoSuchMetalakeException; + +/** Enable metalake. */ +public class MetalakeEnable extends Command { + + private final String metalake; + private Boolean enableAllCatalogs; + + /** + * Enable a metalake + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + * @param metalake The name of the metalake. + * @param enableAllCatalogs Whether to enable all catalogs. + */ + public MetalakeEnable( + String url, boolean ignoreVersions, String metalake, boolean enableAllCatalogs) { + super(url, ignoreVersions); + this.metalake = metalake; + this.enableAllCatalogs = enableAllCatalogs; + } + + /** Enable metalake. */ + @Override + public void handle() { + StringBuilder msgBuilder = new StringBuilder(metalake); + try { + GravitinoAdminClient client = buildAdminClient(); + client.enableMetalake(metalake); + msgBuilder.append(" has been enabled."); + + if (enableAllCatalogs) { + GravitinoMetalake metalakeObject = client.loadMetalake(metalake); + String[] catalogs = metalakeObject.listCatalogs(); + Arrays.stream(catalogs).forEach(metalakeObject::enableCatalog); + msgBuilder.append(" and all catalogs in this metalake have been enabled."); + } + } catch (NoSuchMetalakeException err) { + exitWithError(ErrorMessages.UNKNOWN_METALAKE); + } catch (Exception exp) { + exitWithError(exp.getMessage()); + } + + System.out.println(msgBuilder); + } +} diff --git a/clients/cli/src/main/resources/catalog_help.txt b/clients/cli/src/main/resources/catalog_help.txt index 27ba7eeac34..c29e9dcadbd 100644 --- a/clients/cli/src/main/resources/catalog_help.txt +++ b/clients/cli/src/main/resources/catalog_help.txt @@ -47,4 +47,10 @@ Set a catalog's property gcli catalog set --name catalog_mysql --property test --value value Remove a catalog's property -gcli catalog remove --name catalog_mysql --property test \ No newline at end of file +gcli catalog remove --name catalog_mysql --property test + +Enable a catalog +gcli catalog update -m metalake_demo --name catalog --enable + +Disable a catalog +gcli catalog update -m metalake_demo --name catalog --disable \ No newline at end of file diff --git a/clients/cli/src/main/resources/metalake_help.txt b/clients/cli/src/main/resources/metalake_help.txt index c80d244f521..f700d3a07ea 100644 --- a/clients/cli/src/main/resources/metalake_help.txt +++ b/clients/cli/src/main/resources/metalake_help.txt @@ -38,3 +38,9 @@ gcli metalake set --property test --value value Remove a metalake's property gcli metalake remove --property test + +Enable a metalake +gcli metalake update -m metalake_demo --enable + +Disable a metalke +gcli metalake update -m metalake_demo --disable \ No newline at end of file diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java index eb8bc46d38e..d751d671731 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java @@ -19,17 +19,24 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.util.HashMap; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CatalogAudit; import org.apache.gravitino.cli.commands.CatalogDetails; +import org.apache.gravitino.cli.commands.CatalogDisable; +import org.apache.gravitino.cli.commands.CatalogEnable; import org.apache.gravitino.cli.commands.CreateCatalog; import org.apache.gravitino.cli.commands.DeleteCatalog; import org.apache.gravitino.cli.commands.ListCatalogProperties; @@ -38,6 +45,7 @@ import org.apache.gravitino.cli.commands.SetCatalogProperty; import org.apache.gravitino.cli.commands.UpdateCatalogComment; import org.apache.gravitino.cli.commands.UpdateCatalogName; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -45,10 +53,28 @@ class TestCatalogCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -291,4 +317,92 @@ void testUpdateCatalogNameCommand() { commandLine.handleCommandLine(); verify(mockUpdateName).handle(); } + + @Test + void testEnableCatalogCommand() { + CatalogEnable mockEnable = mock(CatalogEnable.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + when(mockCommandLine.hasOption(GravitinoOptions.ENABLE)).thenReturn(true); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.UPDATE)); + doReturn(mockEnable) + .when(commandLine) + .newCatalogEnable( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", false); + commandLine.handleCommandLine(); + verify(mockEnable).handle(); + } + + @Test + void testEnableCatalogCommandWithRecursive() { + CatalogEnable mockEnable = mock(CatalogEnable.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + when(mockCommandLine.hasOption(GravitinoOptions.ALL)).thenReturn(true); + when(mockCommandLine.hasOption(GravitinoOptions.ENABLE)).thenReturn(true); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.UPDATE)); + doReturn(mockEnable) + .when(commandLine) + .newCatalogEnable( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", true); + commandLine.handleCommandLine(); + verify(mockEnable).handle(); + } + + @Test + void testDisableCatalogCommand() { + CatalogDisable mockDisable = mock(CatalogDisable.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + when(mockCommandLine.hasOption(GravitinoOptions.DISABLE)).thenReturn(true); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.UPDATE)); + doReturn(mockDisable) + .when(commandLine) + .newCatalogDisable(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog"); + commandLine.handleCommandLine(); + verify(mockDisable).handle(); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testCatalogWithDisableAndEnableOptions() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + when(mockCommandLine.hasOption(GravitinoOptions.DISABLE)).thenReturn(true); + when(mockCommandLine.hasOption(GravitinoOptions.ENABLE)).thenReturn(true); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.UPDATE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newCatalogEnable( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", false); + verify(commandLine, never()) + .newCatalogDisable(GravitinoCommandLine.DEFAULT_URL, false, "melake_demo", "catalog"); + assertTrue(errContent.toString().contains("Unable to enable and disable at the same time")); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java index 93de0a6bc9d..377e569aa53 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java @@ -150,6 +150,7 @@ public void catalogWithOneArg() throws ParseException { assertEquals(CommandEntities.CATALOG, entity); } + @Test public void metalakeWithHelpOption() throws ParseException { Options options = new GravitinoOptions().options(); CommandLineParser parser = new DefaultParser(); diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMetalakeCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMetalakeCommands.java index b7468b635a4..01eebb6dab5 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMetalakeCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMetalakeCommands.java @@ -19,12 +19,16 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateMetalake; @@ -33,21 +37,42 @@ import org.apache.gravitino.cli.commands.ListMetalakes; import org.apache.gravitino.cli.commands.MetalakeAudit; import org.apache.gravitino.cli.commands.MetalakeDetails; +import org.apache.gravitino.cli.commands.MetalakeDisable; +import org.apache.gravitino.cli.commands.MetalakeEnable; import org.apache.gravitino.cli.commands.RemoveMetalakeProperty; import org.apache.gravitino.cli.commands.SetMetalakeProperty; import org.apache.gravitino.cli.commands.UpdateMetalakeComment; import org.apache.gravitino.cli.commands.UpdateMetalakeName; +import org.junit.Assert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestMetalakeCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -280,4 +305,80 @@ void testUpdateMetalakeNameForceCommand() { commandLine.handleCommandLine(); verify(mockUpdateName).handle(); } + + @Test + void testEnableMetalakeCommand() { + MetalakeEnable mockEnable = mock(MetalakeEnable.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.ENABLE)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.METALAKE, CommandActions.UPDATE)); + doReturn(mockEnable) + .when(commandLine) + .newMetalakeEnable(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", false); + commandLine.handleCommandLine(); + verify(mockEnable).handle(); + } + + @Test + void testEnableMetalakeCommandWithRecursive() { + MetalakeEnable mockEnable = mock(MetalakeEnable.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.ALL)).thenReturn(true); + when(mockCommandLine.hasOption(GravitinoOptions.ENABLE)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.METALAKE, CommandActions.UPDATE)); + doReturn(mockEnable) + .when(commandLine) + .newMetalakeEnable(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", true); + commandLine.handleCommandLine(); + verify(mockEnable).handle(); + } + + @Test + void testDisableMetalakeCommand() { + MetalakeDisable mockDisable = mock(MetalakeDisable.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.DISABLE)).thenReturn(true); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.METALAKE, CommandActions.UPDATE)); + doReturn(mockDisable) + .when(commandLine) + .newMetalakeDisable(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo"); + + commandLine.handleCommandLine(); + verify(mockDisable).handle(); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testMetalakeWithDisableAndEnableOptions() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.ENABLE)).thenReturn(true); + when(mockCommandLine.hasOption(GravitinoOptions.DISABLE)).thenReturn(true); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.METALAKE, CommandActions.UPDATE)); + + Assert.assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newMetalakeEnable(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", false); + verify(commandLine, never()) + .newMetalakeEnable(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", false); + assertTrue(errContent.toString().contains("Unable to enable and disable at the same time")); + } } diff --git a/docs/cli.md b/docs/cli.md index e6e2f5aa609..64d720f2e8a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -276,6 +276,24 @@ gcli metalake set --property test --value value gcli metalake remove --property test ``` +#### Enable a metalake + +```bash +gcli metalake update -m metalake_demo --enable +``` + +#### Enable a metalake and all catalogs + +```bash +gcli metalake update -m metalake_demo --enable --all +``` + +#### Disable a metalake + +```bash +gcli metalake update -m metalake_demo --disable +``` + ### Catalog commands #### Show all catalogs in a metalake @@ -390,6 +408,24 @@ gcli catalog set --name catalog_mysql --property test --value value gcli catalog remove --name catalog_mysql --property test ``` +#### Enable a catalog + +```bash +gcli catalog update -m metalake_demo --name catalog --enable +``` + +#### Enable a catalog and it's metalake + +```bash +gcli catalog update -m metalake_demo --name catalog --enable --all +``` + +#### Disable a catalog + +```bash +gcli catalog update -m metalake_demo --name catalog --disable +``` + ### Schema commands #### Show all schemas in a catalog From b34b0c8cd56c500b442401a5239edce89ea80ef7 Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Tue, 24 Dec 2024 05:36:24 +0800 Subject: [PATCH 25/47] [#5927] improvement(CLI): fix cli get multiple "Malformed entity name." (#5943) ### What changes were proposed in this pull request? If an entity name is malformed, the CLI should output 'Malformed entity name.' only once, instead of multiple times. ### Why are the changes needed? Fix: #5927 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? ```bash bin/gcli.sh column list -m demo_metalake --name Hive_catalog -i # output: Malformed entity name. ``` --- .../org/apache/gravitino/cli/FullName.java | 20 ++++++++- .../apache/gravitino/cli/TestFulllName.java | 45 +++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java index 46a3bb92dce..a2be2e52c2d 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java @@ -29,6 +29,8 @@ public class FullName { private final CommandLine line; private String metalakeEnv; private boolean matalakeSet = false; + private boolean hasDisplayedMissingNameInfo = true; + private boolean hasDisplayedMalformedInfo = true; /** * Constructor for the {@code FullName} class. @@ -159,14 +161,14 @@ public String getNamePart(int position) { String[] names = line.getOptionValue(GravitinoOptions.NAME).split("\\."); if (names.length <= position) { - System.err.println(ErrorMessages.MALFORMED_NAME); + showMalformedInfo(); return null; } return names[position]; } - System.err.println(ErrorMessages.MISSING_NAME); + showMissingNameInfo(); return null; } @@ -224,4 +226,18 @@ public boolean hasTableName() { public boolean hasColumnName() { return hasNamePart(4); } + + private void showMissingNameInfo() { + if (hasDisplayedMissingNameInfo) { + System.err.println(ErrorMessages.MISSING_NAME); + hasDisplayedMissingNameInfo = false; + } + } + + private void showMalformedInfo() { + if (hasDisplayedMalformedInfo) { + System.err.println(ErrorMessages.MALFORMED_NAME); + hasDisplayedMalformedInfo = false; + } + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java index ecde923a36a..4b5e1fed79b 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java @@ -25,20 +25,37 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.MissingArgumentException; import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class TestFulllName { private Options options; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach public void setUp() { options = new GravitinoOptions().options(); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -152,4 +169,32 @@ public void hasPartNameColumn() throws Exception { assertTrue(fullName.hasTableName()); assertTrue(fullName.hasColumnName()); } + + @Test + @SuppressWarnings("DefaultCharset") + public void testMissingName() throws ParseException { + String[] args = {"column", "list", "-m", "demo_metalake", "-i"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + fullName.getCatalogName(); + fullName.getSchemaName(); + fullName.getTableName(); + fullName.getColumnName(); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_NAME); + } + + @Test + @SuppressWarnings("DefaultCharset") + public void testMalformedName() throws ParseException { + String[] args = {"column", "list", "-m", "demo_metalake", "-i", "--name", "Hive_catalog"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + fullName.getCatalogName(); + fullName.getSchemaName(); + fullName.getTableName(); + fullName.getColumnName(); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MALFORMED_NAME); + } } From a4b7c57307389e3f90584a021f0822e4c57f79b5 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 24 Dec 2024 09:26:24 +0800 Subject: [PATCH 26/47] [#5895] support ADLSToken/AzureAccountKey credential for python client (#5940) ### What changes were proposed in this pull request? Support ADLS credential for python client ### Why are the changes needed? These changes are necessary to support authentication using ADLS credentials and to allow the CredentialFactory to generate ADLS credentials correctly. It ensures proper functionality and integration. Fix: #5895 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Unit tests. --- .../api/credential/adls_token_credential.py | 90 ++++++++++++++ .../azure_account_key_credential.py | 88 ++++++++++++++ .../api/credential/gcs_token_credential.py | 2 +- .../credential/oss_secret_key_credential.py | 16 +-- .../api/credential/oss_token_credential.py | 22 ++-- .../credential/s3_secret_key_credential.py | 16 +-- .../api/credential/s3_token_credential.py | 22 ++-- .../gravitino/utils/credential_factory.py | 39 +++++-- .../unittests/test_credential_factory.py | 110 ++++++++++++++---- 9 files changed, 336 insertions(+), 69 deletions(-) create mode 100644 clients/client-python/gravitino/api/credential/adls_token_credential.py create mode 100644 clients/client-python/gravitino/api/credential/azure_account_key_credential.py diff --git a/clients/client-python/gravitino/api/credential/adls_token_credential.py b/clients/client-python/gravitino/api/credential/adls_token_credential.py new file mode 100644 index 00000000000..40ad0eebbd9 --- /dev/null +++ b/clients/client-python/gravitino/api/credential/adls_token_credential.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC +from typing import Dict + +from gravitino.api.credential.credential import Credential +from gravitino.utils.precondition import Precondition + + +class ADLSTokenCredential(Credential, ABC): + """Represents ADLS token credential.""" + + ADLS_SAS_TOKEN_CREDENTIAL_TYPE: str = "adls-sas-token" + ADLS_DOMAIN: str = "dfs.core.windows.net" + _STORAGE_ACCOUNT_NAME: str = "azure-storage-account-name" + _SAS_TOKEN: str = "adls-sas-token" + + def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int): + self._account_name = credential_info.get(self._STORAGE_ACCOUNT_NAME, None) + self._sas_token = credential_info.get(self._SAS_TOKEN, None) + self._expire_time_in_ms = expire_time_in_ms + Precondition.check_string_not_empty( + self._account_name, "The ADLS account name should not be empty." + ) + Precondition.check_string_not_empty( + self._sas_token, "The ADLS SAS token should not be empty." + ) + Precondition.check_argument( + self._expire_time_in_ms > 0, + "The expiration time of ADLS token credential should be greater than 0", + ) + + def credential_type(self) -> str: + """The type of the credential. + + Returns: + the type of the credential. + """ + return self.ADLS_SAS_TOKEN_CREDENTIAL_TYPE + + def expire_time_in_ms(self) -> int: + """Returns the expiration time of the credential in milliseconds since + the epoch, 0 means it will never expire. + + Returns: + The expiration time of the credential. + """ + return self._expire_time_in_ms + + def credential_info(self) -> Dict[str, str]: + """The credential information. + + Returns: + The credential information. + """ + return { + self._STORAGE_ACCOUNT_NAME: self._account_name, + self._SAS_TOKEN: self._sas_token, + } + + def account_name(self) -> str: + """The ADLS account name. + + Returns: + The ADLS account name. + """ + return self._account_name + + def sas_token(self) -> str: + """The ADLS sas token. + + Returns: + The ADLS sas token. + """ + return self._sas_token diff --git a/clients/client-python/gravitino/api/credential/azure_account_key_credential.py b/clients/client-python/gravitino/api/credential/azure_account_key_credential.py new file mode 100644 index 00000000000..aa60e301548 --- /dev/null +++ b/clients/client-python/gravitino/api/credential/azure_account_key_credential.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC +from typing import Dict + +from gravitino.api.credential.credential import Credential +from gravitino.utils.precondition import Precondition + + +class AzureAccountKeyCredential(Credential, ABC): + """Represents Azure account key credential.""" + + AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE: str = "azure-account-key" + _STORAGE_ACCOUNT_NAME: str = "azure-storage-account-name" + _STORAGE_ACCOUNT_KEY: str = "azure-storage-account-key" + + def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int): + self._account_name = credential_info.get(self._STORAGE_ACCOUNT_NAME, None) + self._account_key = credential_info.get(self._STORAGE_ACCOUNT_KEY, None) + Precondition.check_string_not_empty( + self._account_name, "The Azure account name should not be empty" + ) + Precondition.check_string_not_empty( + self._account_key, "The Azure account key should not be empty" + ) + Precondition.check_argument( + expire_time_in_ms == 0, + "The expiration time of Azure account key credential should be 0", + ) + + def credential_type(self) -> str: + """Returns the type of the credential. + + Returns: + The type of the credential. + """ + return self.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE + + def expire_time_in_ms(self) -> int: + """Returns the expiration time of the credential in milliseconds since + the epoch, 0 means it will never expire. + + Returns: + The expiration time of the credential. + """ + return 0 + + def credential_info(self) -> Dict[str, str]: + """The credential information. + + Returns: + The credential information. + """ + return { + self._STORAGE_ACCOUNT_NAME: self._account_name, + self._STORAGE_ACCOUNT_KEY: self._account_key, + } + + def account_name(self) -> str: + """The Azure account name. + + Returns: + The Azure account name. + """ + return self._account_name + + def account_key(self) -> str: + """The Azure account key. + + Returns: + The Azure account key. + """ + return self._account_key diff --git a/clients/client-python/gravitino/api/credential/gcs_token_credential.py b/clients/client-python/gravitino/api/credential/gcs_token_credential.py index 1362383f0bb..0221ac07ca9 100644 --- a/clients/client-python/gravitino/api/credential/gcs_token_credential.py +++ b/clients/client-python/gravitino/api/credential/gcs_token_credential.py @@ -31,7 +31,7 @@ class GCSTokenCredential(Credential, ABC): _expire_time_in_ms: int = 0 def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int): - self._token = credential_info[self._GCS_TOKEN_NAME] + self._token = credential_info.get(self._GCS_TOKEN_NAME, None) self._expire_time_in_ms = expire_time_in_ms Precondition.check_string_not_empty( self._token, "GCS token should not be empty" diff --git a/clients/client-python/gravitino/api/credential/oss_secret_key_credential.py b/clients/client-python/gravitino/api/credential/oss_secret_key_credential.py index 919a3782ef9..69a9646490e 100644 --- a/clients/client-python/gravitino/api/credential/oss_secret_key_credential.py +++ b/clients/client-python/gravitino/api/credential/oss_secret_key_credential.py @@ -26,14 +26,14 @@ class OSSSecretKeyCredential(Credential, ABC): """Represents OSS secret key credential.""" OSS_SECRET_KEY_CREDENTIAL_TYPE: str = "oss-secret-key" - _GRAVITINO_OSS_STATIC_ACCESS_KEY_ID: str = "oss-access-key-id" - _GRAVITINO_OSS_STATIC_SECRET_ACCESS_KEY: str = "oss-secret-access-key" + _STATIC_ACCESS_KEY_ID: str = "oss-access-key-id" + _STATIC_SECRET_ACCESS_KEY: str = "oss-secret-access-key" def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int): - self._access_key_id = credential_info[self._GRAVITINO_OSS_STATIC_ACCESS_KEY_ID] - self._secret_access_key = credential_info[ - self._GRAVITINO_OSS_STATIC_SECRET_ACCESS_KEY - ] + self._access_key_id = credential_info.get(self._STATIC_ACCESS_KEY_ID, None) + self._secret_access_key = credential_info.get( + self._STATIC_SECRET_ACCESS_KEY, None + ) Precondition.check_string_not_empty( self._access_key_id, "The OSS access key ID should not be empty" ) @@ -69,8 +69,8 @@ def credential_info(self) -> Dict[str, str]: The credential information. """ return { - self._GRAVITINO_OSS_STATIC_SECRET_ACCESS_KEY: self._secret_access_key, - self._GRAVITINO_OSS_STATIC_ACCESS_KEY_ID: self._access_key_id, + self._STATIC_ACCESS_KEY_ID: self._access_key_id, + self._STATIC_SECRET_ACCESS_KEY: self._secret_access_key, } def access_key_id(self) -> str: diff --git a/clients/client-python/gravitino/api/credential/oss_token_credential.py b/clients/client-python/gravitino/api/credential/oss_token_credential.py index 70dad14a1aa..d217ad8c896 100644 --- a/clients/client-python/gravitino/api/credential/oss_token_credential.py +++ b/clients/client-python/gravitino/api/credential/oss_token_credential.py @@ -26,16 +26,16 @@ class OSSTokenCredential(Credential, ABC): """Represents OSS token credential.""" OSS_TOKEN_CREDENTIAL_TYPE: str = "oss-token" - _GRAVITINO_OSS_SESSION_ACCESS_KEY_ID: str = "oss-access-key-id" - _GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY: str = "oss-secret-access-key" - _GRAVITINO_OSS_TOKEN: str = "oss-security-token" + _STATIC_ACCESS_KEY_ID: str = "oss-access-key-id" + _STATIC_SECRET_ACCESS_KEY: str = "oss-secret-access-key" + _OSS_TOKEN: str = "oss-security-token" def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int): - self._access_key_id = credential_info[self._GRAVITINO_OSS_SESSION_ACCESS_KEY_ID] - self._secret_access_key = credential_info[ - self._GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY - ] - self._security_token = credential_info[self._GRAVITINO_OSS_TOKEN] + self._access_key_id = credential_info.get(self._STATIC_ACCESS_KEY_ID, None) + self._secret_access_key = credential_info.get( + self._STATIC_SECRET_ACCESS_KEY, None + ) + self._security_token = credential_info.get(self._OSS_TOKEN, None) self._expire_time_in_ms = expire_time_in_ms Precondition.check_string_not_empty( self._access_key_id, "The OSS access key ID should not be empty" @@ -75,9 +75,9 @@ def credential_info(self) -> Dict[str, str]: The credential information. """ return { - self._GRAVITINO_OSS_TOKEN: self._security_token, - self._GRAVITINO_OSS_SESSION_ACCESS_KEY_ID: self._access_key_id, - self._GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY: self._secret_access_key, + self._STATIC_ACCESS_KEY_ID: self._access_key_id, + self._STATIC_SECRET_ACCESS_KEY: self._secret_access_key, + self._OSS_TOKEN: self._security_token, } def access_key_id(self) -> str: diff --git a/clients/client-python/gravitino/api/credential/s3_secret_key_credential.py b/clients/client-python/gravitino/api/credential/s3_secret_key_credential.py index 735c41e2ee0..05c221fe2a8 100644 --- a/clients/client-python/gravitino/api/credential/s3_secret_key_credential.py +++ b/clients/client-python/gravitino/api/credential/s3_secret_key_credential.py @@ -26,14 +26,14 @@ class S3SecretKeyCredential(Credential, ABC): """Represents S3 secret key credential.""" S3_SECRET_KEY_CREDENTIAL_TYPE: str = "s3-secret-key" - _GRAVITINO_S3_STATIC_ACCESS_KEY_ID: str = "s3-access-key-id" - _GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY: str = "s3-secret-access-key" + _STATIC_ACCESS_KEY_ID: str = "s3-access-key-id" + _STATIC_SECRET_ACCESS_KEY: str = "s3-secret-access-key" def __init__(self, credential_info: Dict[str, str], expire_time: int): - self._access_key_id = credential_info[self._GRAVITINO_S3_STATIC_ACCESS_KEY_ID] - self._secret_access_key = credential_info[ - self._GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY - ] + self._access_key_id = credential_info.get(self._STATIC_ACCESS_KEY_ID, None) + self._secret_access_key = credential_info.get( + self._STATIC_SECRET_ACCESS_KEY, None + ) Precondition.check_string_not_empty( self._access_key_id, "S3 access key id should not be empty" ) @@ -70,8 +70,8 @@ def credential_info(self) -> Dict[str, str]: The credential information. """ return { - self._GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY: self._secret_access_key, - self._GRAVITINO_S3_STATIC_ACCESS_KEY_ID: self._access_key_id, + self._STATIC_ACCESS_KEY_ID: self._access_key_id, + self._STATIC_SECRET_ACCESS_KEY: self._secret_access_key, } def access_key_id(self) -> str: diff --git a/clients/client-python/gravitino/api/credential/s3_token_credential.py b/clients/client-python/gravitino/api/credential/s3_token_credential.py index c72d9f02a7d..d95919f6628 100644 --- a/clients/client-python/gravitino/api/credential/s3_token_credential.py +++ b/clients/client-python/gravitino/api/credential/s3_token_credential.py @@ -26,9 +26,9 @@ class S3TokenCredential(Credential, ABC): """Represents the S3 token credential.""" S3_TOKEN_CREDENTIAL_TYPE: str = "s3-token" - _GRAVITINO_S3_SESSION_ACCESS_KEY_ID: str = "s3-access-key-id" - _GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY: str = "s3-secret-access-key" - _GRAVITINO_S3_TOKEN: str = "s3-session-token" + _SESSION_ACCESS_KEY_ID: str = "s3-access-key-id" + _SESSION_SECRET_ACCESS_KEY: str = "s3-secret-access-key" + _SESSION_TOKEN: str = "s3-session-token" _expire_time_in_ms: int = 0 _access_key_id: str = None @@ -36,11 +36,11 @@ class S3TokenCredential(Credential, ABC): _session_token: str = None def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int): - self._access_key_id = credential_info[self._GRAVITINO_S3_SESSION_ACCESS_KEY_ID] - self._secret_access_key = credential_info[ - self._GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY - ] - self._session_token = credential_info[self._GRAVITINO_S3_TOKEN] + self._access_key_id = credential_info.get(self._SESSION_ACCESS_KEY_ID, None) + self._secret_access_key = credential_info.get( + self._SESSION_SECRET_ACCESS_KEY, None + ) + self._session_token = credential_info.get(self._SESSION_TOKEN, None) self._expire_time_in_ms = expire_time_in_ms Precondition.check_string_not_empty( self._access_key_id, "The S3 access key ID should not be empty" @@ -80,9 +80,9 @@ def credential_info(self) -> Dict[str, str]: The credential information. """ return { - self._GRAVITINO_S3_TOKEN: self._session_token, - self._GRAVITINO_S3_SESSION_ACCESS_KEY_ID: self._access_key_id, - self._GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY: self._secret_access_key, + self._SESSION_ACCESS_KEY_ID: self._access_key_id, + self._SESSION_SECRET_ACCESS_KEY: self._secret_access_key, + self._SESSION_TOKEN: self._session_token, } def access_key_id(self) -> str: diff --git a/clients/client-python/gravitino/utils/credential_factory.py b/clients/client-python/gravitino/utils/credential_factory.py index 7a584caa3e6..32d7465b806 100644 --- a/clients/client-python/gravitino/utils/credential_factory.py +++ b/clients/client-python/gravitino/utils/credential_factory.py @@ -16,12 +16,17 @@ # under the License. from typing import Dict + from gravitino.api.credential.credential import Credential from gravitino.api.credential.gcs_token_credential import GCSTokenCredential from gravitino.api.credential.oss_token_credential import OSSTokenCredential from gravitino.api.credential.s3_secret_key_credential import S3SecretKeyCredential from gravitino.api.credential.s3_token_credential import S3TokenCredential from gravitino.api.credential.oss_secret_key_credential import OSSSecretKeyCredential +from gravitino.api.credential.adls_token_credential import ADLSTokenCredential +from gravitino.api.credential.azure_account_key_credential import ( + AzureAccountKeyCredential, +) class CredentialFactory: @@ -29,14 +34,28 @@ class CredentialFactory: def create( credential_type: str, credential_info: Dict[str, str], expire_time_in_ms: int ) -> Credential: + credential = None + if credential_type == S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE: - return S3TokenCredential(credential_info, expire_time_in_ms) - if credential_type == S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE: - return S3SecretKeyCredential(credential_info, expire_time_in_ms) - if credential_type == GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE: - return GCSTokenCredential(credential_info, expire_time_in_ms) - if credential_type == OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE: - return OSSTokenCredential(credential_info, expire_time_in_ms) - if credential_type == OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE: - return OSSSecretKeyCredential(credential_info, expire_time_in_ms) - raise NotImplementedError(f"Credential type {credential_type} is not supported") + credential = S3TokenCredential(credential_info, expire_time_in_ms) + elif credential_type == S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE: + credential = S3SecretKeyCredential(credential_info, expire_time_in_ms) + elif credential_type == GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE: + credential = GCSTokenCredential(credential_info, expire_time_in_ms) + elif credential_type == OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE: + credential = OSSTokenCredential(credential_info, expire_time_in_ms) + elif credential_type == OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE: + credential = OSSSecretKeyCredential(credential_info, expire_time_in_ms) + elif credential_type == ADLSTokenCredential.ADLS_SAS_TOKEN_CREDENTIAL_TYPE: + credential = ADLSTokenCredential(credential_info, expire_time_in_ms) + elif ( + credential_type + == AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE + ): + credential = AzureAccountKeyCredential(credential_info, expire_time_in_ms) + else: + raise NotImplementedError( + f"Credential type {credential_type} is not supported" + ) + + return credential diff --git a/clients/client-python/tests/unittests/test_credential_factory.py b/clients/client-python/tests/unittests/test_credential_factory.py index 94fd02d1df2..4c4a91495a1 100644 --- a/clients/client-python/tests/unittests/test_credential_factory.py +++ b/clients/client-python/tests/unittests/test_credential_factory.py @@ -25,15 +25,19 @@ from gravitino.api.credential.s3_token_credential import S3TokenCredential from gravitino.utils.credential_factory import CredentialFactory from gravitino.api.credential.oss_secret_key_credential import OSSSecretKeyCredential +from gravitino.api.credential.adls_token_credential import ADLSTokenCredential +from gravitino.api.credential.azure_account_key_credential import ( + AzureAccountKeyCredential, +) class TestCredentialFactory(unittest.TestCase): def test_s3_token_credential(self): s3_credential_info = { - S3TokenCredential._GRAVITINO_S3_SESSION_ACCESS_KEY_ID: "access_key", - S3TokenCredential._GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY: "secret_key", - S3TokenCredential._GRAVITINO_S3_TOKEN: "session_token", + S3TokenCredential._SESSION_ACCESS_KEY_ID: "access_key", + S3TokenCredential._SESSION_SECRET_ACCESS_KEY: "secret_key", + S3TokenCredential._SESSION_TOKEN: "session_token", } s3_credential = S3TokenCredential(s3_credential_info, 1000) credential_info = s3_credential.credential_info() @@ -42,6 +46,12 @@ def test_s3_token_credential(self): check_credential = CredentialFactory.create( s3_credential.S3_TOKEN_CREDENTIAL_TYPE, credential_info, expire_time ) + self.assertEqual( + S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE, + check_credential.credential_type(), + ) + + self.assertIsInstance(check_credential, S3TokenCredential) self.assertEqual("access_key", check_credential.access_key_id()) self.assertEqual("secret_key", check_credential.secret_access_key()) self.assertEqual("session_token", check_credential.session_token()) @@ -49,8 +59,8 @@ def test_s3_token_credential(self): def test_s3_secret_key_credential(self): s3_credential_info = { - S3SecretKeyCredential._GRAVITINO_S3_STATIC_ACCESS_KEY_ID: "access_key", - S3SecretKeyCredential._GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY: "secret_key", + S3SecretKeyCredential._STATIC_ACCESS_KEY_ID: "access_key", + S3SecretKeyCredential._STATIC_SECRET_ACCESS_KEY: "secret_key", } s3_credential = S3SecretKeyCredential(s3_credential_info, 0) credential_info = s3_credential.credential_info() @@ -59,43 +69,53 @@ def test_s3_secret_key_credential(self): check_credential = CredentialFactory.create( s3_credential.S3_SECRET_KEY_CREDENTIAL_TYPE, credential_info, expire_time ) + self.assertEqual( + S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE, + check_credential.credential_type(), + ) + + self.assertIsInstance(check_credential, S3SecretKeyCredential) self.assertEqual("access_key", check_credential.access_key_id()) self.assertEqual("secret_key", check_credential.secret_access_key()) self.assertEqual(0, check_credential.expire_time_in_ms()) def test_gcs_token_credential(self): - credential_info = {GCSTokenCredential._GCS_TOKEN_NAME: "token"} - credential = GCSTokenCredential(credential_info, 1000) - credential_info = credential.credential_info() - expire_time = credential.expire_time_in_ms() + gcs_credential_info = {GCSTokenCredential._GCS_TOKEN_NAME: "token"} + gcs_credential = GCSTokenCredential(gcs_credential_info, 1000) + credential_info = gcs_credential.credential_info() + expire_time = gcs_credential.expire_time_in_ms() check_credential = CredentialFactory.create( - credential.credential_type(), credential_info, expire_time + gcs_credential.credential_type(), credential_info, expire_time ) self.assertEqual( GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE, check_credential.credential_type(), ) + + self.assertIsInstance(check_credential, GCSTokenCredential) self.assertEqual("token", check_credential.token()) self.assertEqual(1000, check_credential.expire_time_in_ms()) def test_oss_token_credential(self): - credential_info = { - OSSTokenCredential._GRAVITINO_OSS_TOKEN: "token", - OSSTokenCredential._GRAVITINO_OSS_SESSION_ACCESS_KEY_ID: "access_id", - OSSTokenCredential._GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY: "secret_key", + oss_credential_info = { + OSSTokenCredential._STATIC_ACCESS_KEY_ID: "access_id", + OSSTokenCredential._STATIC_SECRET_ACCESS_KEY: "secret_key", + OSSTokenCredential._OSS_TOKEN: "token", } - credential = OSSTokenCredential(credential_info, 1000) - credential_info = credential.credential_info() - expire_time = credential.expire_time_in_ms() + oss_credential = OSSTokenCredential(oss_credential_info, 1000) + credential_info = oss_credential.credential_info() + expire_time = oss_credential.expire_time_in_ms() check_credential = CredentialFactory.create( - credential.credential_type(), credential_info, expire_time + oss_credential.credential_type(), credential_info, expire_time ) self.assertEqual( OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE, check_credential.credential_type(), ) + + self.assertIsInstance(check_credential, OSSTokenCredential) self.assertEqual("token", check_credential.security_token()) self.assertEqual("access_id", check_credential.access_key_id()) self.assertEqual("secret_key", check_credential.secret_access_key()) @@ -103,8 +123,8 @@ def test_oss_token_credential(self): def test_oss_secret_key_credential(self): oss_credential_info = { - OSSSecretKeyCredential._GRAVITINO_OSS_STATIC_ACCESS_KEY_ID: "access_key", - OSSSecretKeyCredential._GRAVITINO_OSS_STATIC_SECRET_ACCESS_KEY: "secret_key", + OSSSecretKeyCredential._STATIC_ACCESS_KEY_ID: "access_key", + OSSSecretKeyCredential._STATIC_SECRET_ACCESS_KEY: "secret_key", } oss_credential = OSSSecretKeyCredential(oss_credential_info, 0) credential_info = oss_credential.credential_info() @@ -113,6 +133,56 @@ def test_oss_secret_key_credential(self): check_credential = CredentialFactory.create( oss_credential.OSS_SECRET_KEY_CREDENTIAL_TYPE, credential_info, expire_time ) + self.assertEqual( + OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE, + check_credential.credential_type(), + ) + + self.assertIsInstance(check_credential, OSSSecretKeyCredential) self.assertEqual("access_key", check_credential.access_key_id()) self.assertEqual("secret_key", check_credential.secret_access_key()) self.assertEqual(0, check_credential.expire_time_in_ms()) + + def test_adls_token_credential(self): + adls_credential_info = { + ADLSTokenCredential._STORAGE_ACCOUNT_NAME: "account_name", + ADLSTokenCredential._SAS_TOKEN: "sas_token", + } + adls_credential = ADLSTokenCredential(adls_credential_info, 1000) + credential_info = adls_credential.credential_info() + expire_time = adls_credential.expire_time_in_ms() + + check_credential = CredentialFactory.create( + adls_credential.credential_type(), credential_info, expire_time + ) + self.assertEqual( + ADLSTokenCredential.ADLS_SAS_TOKEN_CREDENTIAL_TYPE, + check_credential.credential_type(), + ) + + self.assertIsInstance(check_credential, ADLSTokenCredential) + self.assertEqual("account_name", check_credential.account_name()) + self.assertEqual("sas_token", check_credential.sas_token()) + self.assertEqual(1000, check_credential.expire_time_in_ms()) + + def test_azure_account_key_credential(self): + azure_credential_info = { + AzureAccountKeyCredential._STORAGE_ACCOUNT_NAME: "account_name", + AzureAccountKeyCredential._STORAGE_ACCOUNT_KEY: "account_key", + } + azure_credential = AzureAccountKeyCredential(azure_credential_info, 0) + credential_info = azure_credential.credential_info() + expire_time = azure_credential.expire_time_in_ms() + + check_credential = CredentialFactory.create( + azure_credential.credential_type(), credential_info, expire_time + ) + self.assertEqual( + AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE, + check_credential.credential_type(), + ) + + self.assertIsInstance(check_credential, AzureAccountKeyCredential) + self.assertEqual("account_name", check_credential.account_name()) + self.assertEqual("account_key", check_credential.account_key()) + self.assertEqual(0, check_credential.expire_time_in_ms()) From a4f6afb628d54049d33f1de20f215c01b3a65992 Mon Sep 17 00:00:00 2001 From: Xun Date: Tue, 24 Dec 2024 16:19:08 +0800 Subject: [PATCH 27/47] [#5969] fix(Docker): Failed to create Docker network by concurrent execution ITs (#5970) ### What changes were proposed in this pull request? Uses thread safe to create Docker network ### Why are the changes needed? Fix: #5969 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? CI Passed. --- .../test/container/ContainerSuite.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java index 5745cc6d08f..d2a5ee6152b 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java @@ -86,22 +86,24 @@ public class ContainerSuite implements Closeable { protected static final CloseableGroup closer = CloseableGroup.create(); private static void initIfNecessary() { - if (initialized) { - return; - } - - try { - // Check if docker is available and you should never close the global DockerClient! - DockerClient dockerClient = DockerClientFactory.instance().client(); - Info info = dockerClient.infoCmd().exec(); - LOG.info("Docker info: {}", info); - - if ("true".equalsIgnoreCase(System.getenv("NEED_CREATE_DOCKER_NETWORK"))) { - network = createDockerNetwork(); + if (!initialized) { + synchronized (ContainerSuite.class) { + if (!initialized) { + try { + // Check if docker is available and you should never close the global DockerClient! + DockerClient dockerClient = DockerClientFactory.instance().client(); + Info info = dockerClient.infoCmd().exec(); + LOG.info("Docker info: {}", info); + + if ("true".equalsIgnoreCase(System.getenv("NEED_CREATE_DOCKER_NETWORK"))) { + network = createDockerNetwork(); + } + initialized = true; + } catch (Exception e) { + throw new RuntimeException("Failed to initialize ContainerSuite", e); + } + } } - initialized = true; - } catch (Exception e) { - throw new RuntimeException("Failed to initialize ContainerSuite", e); } } From 3d6f5f2a332ba57ea6273a1a2896e4323aebce6f Mon Sep 17 00:00:00 2001 From: roryqi Date: Tue, 24 Dec 2024 16:22:25 +0800 Subject: [PATCH 28/47] [#5661] feat(auth): Add JDBC authorization plugin interface (#5904) ### What changes were proposed in this pull request? Add JDBC authorization plugin interface ### Why are the changes needed? Fix: #5661 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add a UT --- .../access-control-integration-test.yml | 3 + .../authorization-jdbc/build.gradle.kts | 94 ++++ .../jdbc/JdbcAuthorizationPlugin.java | 461 ++++++++++++++++++ .../jdbc/JdbcAuthorizationProperties.java | 44 ++ .../jdbc/JdbcAuthorizationSQL.java | 117 +++++ .../jdbc/JdbcMetadataObject.java | 106 ++++ .../authorization/jdbc/JdbcPrivilege.java | 55 +++ .../jdbc/JdbcSecurableObject.java | 65 +++ .../JdbcSecurableObjectMappingProvider.java | 212 ++++++++ .../jdbc/JdbcAuthorizationPluginTest.java | 317 ++++++++++++ settings.gradle.kts | 2 +- 11 files changed, 1475 insertions(+), 1 deletion(-) create mode 100644 authorizations/authorization-jdbc/build.gradle.kts create mode 100644 authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java create mode 100644 authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java create mode 100644 authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java create mode 100644 authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java create mode 100644 authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java create mode 100644 authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java create mode 100644 authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java create mode 100644 authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index 54ffde2ee82..6997eaf9a4c 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -90,6 +90,9 @@ jobs: ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test + ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test + ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test + ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/authorizations/authorization-jdbc/build.gradle.kts b/authorizations/authorization-jdbc/build.gradle.kts new file mode 100644 index 00000000000..8b105908c26 --- /dev/null +++ b/authorizations/authorization-jdbc/build.gradle.kts @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "authorization-jdbc" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +dependencies { + implementation(project(":api")) { + exclude(group = "*") + } + implementation(project(":core")) { + exclude(group = "*") + } + + implementation(libs.bundles.log4j) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.javax.jaxb.api) { + exclude("*") + } + implementation(libs.javax.ws.rs.api) + implementation(libs.jettison) + compileOnly(libs.lombok) + implementation(libs.mail) + implementation(libs.rome) + implementation(libs.commons.dbcp2) + + testImplementation(project(":common")) + testImplementation(project(":clients:client-java")) + testImplementation(project(":server")) + testImplementation(project(":catalogs:catalog-common")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.mockito.core) + testImplementation(libs.testcontainers) + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks { + val runtimeJars by registering(Copy::class) { + from(configurations.runtimeClasspath) + into("build/libs") + } + + val copyAuthorizationLibs by registering(Copy::class) { + dependsOn("jar", runtimeJars) + from("build/libs") { + exclude("guava-*.jar") + exclude("log4j-*.jar") + exclude("slf4j-*.jar") + } + into("$rootDir/distribution/package/authorizations/ranger/libs") + } + + register("copyLibAndConfig", Copy::class) { + dependsOn(copyAuthorizationLibs) + } + + jar { + dependsOn(runtimeJars) + } +} + +tasks.test { + dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars") + + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/test/**") + } else { + dependsOn(tasks.jar) + } +} diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java new file mode 100644 index 00000000000..f889cee2240 --- /dev/null +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java @@ -0,0 +1,461 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.jdbc; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.pool2.impl.BaseObjectPoolConfig; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.annotation.Unstable; +import org.apache.gravitino.authorization.AuthorizationPrivilege; +import org.apache.gravitino.authorization.AuthorizationSecurableObject; +import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.MetadataObjectChange; +import org.apache.gravitino.authorization.Owner; +import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.RoleChange; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.User; +import org.apache.gravitino.connector.authorization.AuthorizationPlugin; +import org.apache.gravitino.exceptions.AuthorizationPluginException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.GroupEntity; +import org.apache.gravitino.meta.UserEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JdbcSQLBasedAuthorizationPlugin is the base class for all JDBC-based authorization plugins. For + * example, JdbcHiveAuthorizationPlugin is the JDBC-based authorization plugin for Hive. Different + * JDBC-based authorization plugins can inherit this class and implement their own SQL statements. + */ +@Unstable +abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL { + + private static final String GROUP_PREFIX = "GRAVITINO_GROUP_"; + private static final Logger LOG = LoggerFactory.getLogger(JdbcAuthorizationPlugin.class); + + protected BasicDataSource dataSource; + protected JdbcSecurableObjectMappingProvider mappingProvider; + + public JdbcAuthorizationPlugin(Map config) { + // Initialize the data source + dataSource = new BasicDataSource(); + JdbcAuthorizationProperties.validate(config); + + String jdbcUrl = config.get(JdbcAuthorizationProperties.JDBC_URL); + dataSource.setUrl(jdbcUrl); + dataSource.setDriverClassName(config.get(JdbcAuthorizationProperties.JDBC_DRIVER)); + dataSource.setUsername(config.get(JdbcAuthorizationProperties.JDBC_USERNAME)); + dataSource.setPassword(config.get(JdbcAuthorizationProperties.JDBC_PASSWORD)); + dataSource.setDefaultAutoCommit(true); + dataSource.setMaxTotal(20); + dataSource.setMaxIdle(5); + dataSource.setMinIdle(0); + dataSource.setLogAbandoned(true); + dataSource.setRemoveAbandonedOnBorrow(true); + dataSource.setTestOnBorrow(BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW); + dataSource.setTestWhileIdle(BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE); + dataSource.setNumTestsPerEvictionRun(BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN); + dataSource.setTestOnReturn(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN); + dataSource.setLifo(BaseObjectPoolConfig.DEFAULT_LIFO); + mappingProvider = new JdbcSecurableObjectMappingProvider(); + } + + @Override + public void close() throws IOException { + if (dataSource != null) { + try { + dataSource.close(); + dataSource = null; + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws RuntimeException { + // This interface mainly handles the metadata object rename change and delete change. + // The privilege for JdbcSQLBasedAuthorizationPlugin will be renamed or deleted automatically. + // We don't need to do any other things. + return true; + } + + @Override + public Boolean onRoleCreated(Role role) throws AuthorizationPluginException { + List sqls = getCreateRoleSQL(role.name()); + for (String sql : sqls) { + executeUpdateSQL(sql, "already exists"); + } + + if (role.securableObjects() != null) { + for (SecurableObject object : role.securableObjects()) { + onRoleUpdated(role, RoleChange.addSecurableObject(role.name(), object)); + } + } + + return true; + } + + @Override + public Boolean onRoleAcquired(Role role) throws AuthorizationPluginException { + throw new UnsupportedOperationException("Doesn't support to acquired a role"); + } + + @Override + public Boolean onRoleDeleted(Role role) throws AuthorizationPluginException { + List sqls = getDropRoleSQL(role.name()); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + return null; + } + + @Override + public Boolean onRoleUpdated(Role role, RoleChange... changes) + throws AuthorizationPluginException { + onRoleCreated(role); + for (RoleChange change : changes) { + if (change instanceof RoleChange.AddSecurableObject) { + SecurableObject object = ((RoleChange.AddSecurableObject) change).getSecurableObject(); + grantObjectPrivileges(role, object); + } else if (change instanceof RoleChange.RemoveSecurableObject) { + SecurableObject object = ((RoleChange.RemoveSecurableObject) change).getSecurableObject(); + revokeObjectPrivileges(role, object); + } else if (change instanceof RoleChange.UpdateSecurableObject) { + RoleChange.UpdateSecurableObject updateChange = (RoleChange.UpdateSecurableObject) change; + SecurableObject addObject = updateChange.getNewSecurableObject(); + SecurableObject removeObject = updateChange.getSecurableObject(); + revokeObjectPrivileges(role, removeObject); + grantObjectPrivileges(role, addObject); + } else { + throw new IllegalArgumentException( + String.format("RoleChange is not supported - %s", change)); + } + } + return true; + } + + @Override + public Boolean onGrantedRolesToUser(List roles, User user) + throws AuthorizationPluginException { + + for (Role role : roles) { + onRoleCreated(role); + List sqls = getGrantRoleSQL(role.name(), "USER", user.name()); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + } + return true; + } + + @Override + public Boolean onRevokedRolesFromUser(List roles, User user) + throws AuthorizationPluginException { + + for (Role role : roles) { + onRoleCreated(role); + List sqls = getRevokeRoleSQL(role.name(), "USER", user.name()); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + } + return true; + } + + @Override + public Boolean onGrantedRolesToGroup(List roles, Group group) + throws AuthorizationPluginException { + + for (Role role : roles) { + onRoleCreated(role); + List sqls = + getGrantRoleSQL(role.name(), "USER", String.format("%s%s", GROUP_PREFIX, group.name())); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + } + return true; + } + + @Override + public Boolean onRevokedRolesFromGroup(List roles, Group group) + throws AuthorizationPluginException { + + for (Role role : roles) { + onRoleCreated(role); + List sqls = + getRevokeRoleSQL(role.name(), "USER", String.format("%s%s", GROUP_PREFIX, group.name())); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + } + return true; + } + + @Override + public Boolean onUserAdded(User user) throws AuthorizationPluginException { + List sqls = getCreateUserSQL(user.name()); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + return true; + } + + @Override + public Boolean onUserRemoved(User user) throws AuthorizationPluginException { + List sqls = getDropUserSQL(user.name()); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + return true; + } + + @Override + public Boolean onUserAcquired(User user) throws AuthorizationPluginException { + throw new UnsupportedOperationException("Doesn't support to acquired a user"); + } + + @Override + public Boolean onGroupAdded(Group group) throws AuthorizationPluginException { + String name = String.format("%s%s", GROUP_PREFIX, group.name()); + List sqls = getCreateUserSQL(name); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + return true; + } + + @Override + public Boolean onGroupRemoved(Group group) throws AuthorizationPluginException { + String name = String.format("%s%s", GROUP_PREFIX, group.name()); + List sqls = getDropUserSQL(name); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + return true; + } + + @Override + public Boolean onGroupAcquired(Group group) throws AuthorizationPluginException { + throw new UnsupportedOperationException("Doesn't support to acquired a group"); + } + + @Override + public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner) + throws AuthorizationPluginException { + if (newOwner.type() == Owner.Type.USER) { + onUserAdded( + UserEntity.builder() + .withName(newOwner.name()) + .withId(0L) + .withAuditInfo(AuditInfo.EMPTY) + .build()); + } else if (newOwner.type() == Owner.Type.GROUP) { + onGroupAdded( + GroupEntity.builder() + .withName(newOwner.name()) + .withId(0L) + .withAuditInfo(AuditInfo.EMPTY) + .build()); + } else { + throw new IllegalArgumentException( + String.format("Don't support owner type %s", newOwner.type())); + } + + List authObjects = mappingProvider.translateOwner(metadataObject); + for (AuthorizationSecurableObject authObject : authObjects) { + List sqls = + getSetOwnerSQL( + authObject.type().metadataObjectType(), authObject.fullName(), preOwner, newOwner); + for (String sql : sqls) { + executeUpdateSQL(sql); + } + } + return true; + } + + @Override + public List getCreateUserSQL(String username) { + return Lists.newArrayList(String.format("CREATE USER %s", username)); + } + + @Override + public List getDropUserSQL(String username) { + return Lists.newArrayList(String.format("DROP USER %s", username)); + } + + @Override + public List getCreateRoleSQL(String roleName) { + return Lists.newArrayList(String.format("CREATE ROLE %s", roleName)); + } + + @Override + public List getDropRoleSQL(String roleName) { + return Lists.newArrayList(String.format("DROP ROLE %s", roleName)); + } + + @Override + public List getGrantPrivilegeSQL( + String privilege, String objectType, String objectName, String roleName) { + return Lists.newArrayList( + String.format("GRANT %s ON %s %s TO ROLE %s", privilege, objectType, objectName, roleName)); + } + + @Override + public List getRevokePrivilegeSQL( + String privilege, String objectType, String objectName, String roleName) { + return Lists.newArrayList( + String.format( + "REVOKE %s ON %s %s FROM ROLE %s", privilege, objectType, objectName, roleName)); + } + + @Override + public List getGrantRoleSQL(String roleName, String grantorType, String grantorName) { + return Lists.newArrayList( + String.format("GRANT ROLE %s TO %s %s", roleName, grantorType, grantorName)); + } + + @Override + public List getRevokeRoleSQL(String roleName, String revokerType, String revokerName) { + return Lists.newArrayList( + String.format("REVOKE ROLE %s FROM %s %s", roleName, revokerType, revokerName)); + } + + @VisibleForTesting + Connection getConnection() throws SQLException { + return dataSource.getConnection(); + } + + protected void executeUpdateSQL(String sql) { + executeUpdateSQL(sql, null); + } + + /** + * Convert the object name contains `*` to a list of AuthorizationSecurableObject. + * + * @param object The object contains the name with `*` to be converted + * @return The list of AuthorizationSecurableObject + */ + protected List convertResourceAll( + AuthorizationSecurableObject object) { + List authObjects = Lists.newArrayList(); + authObjects.add(object); + return authObjects; + } + + protected List filterUnsupportedPrivileges( + List privileges) { + return privileges; + } + + protected AuthorizationPluginException toAuthorizationPluginException(SQLException se) { + return new AuthorizationPluginException( + "JDBC authorization plugin fail to execute SQL, error code: %d", se.getErrorCode()); + } + + void executeUpdateSQL(String sql, String ignoreErrorMsg) { + try (final Connection connection = getConnection()) { + try (final Statement statement = connection.createStatement()) { + statement.executeUpdate(sql); + } + } catch (SQLException se) { + if (ignoreErrorMsg != null && se.getMessage().contains(ignoreErrorMsg)) { + return; + } + LOG.error("JDBC authorization plugin exception: ", se); + throw toAuthorizationPluginException(se); + } + } + + private void grantObjectPrivileges(Role role, SecurableObject object) { + List authObjects = mappingProvider.translatePrivilege(object); + for (AuthorizationSecurableObject authObject : authObjects) { + List convertedObjects = Lists.newArrayList(); + if (authObject.name().equals(JdbcSecurableObject.ALL)) { + convertedObjects.addAll(convertResourceAll(authObject)); + } else { + convertedObjects.add(authObject); + } + + for (AuthorizationSecurableObject convertedObject : convertedObjects) { + List privileges = + filterUnsupportedPrivileges(authObject.privileges()).stream() + .map(AuthorizationPrivilege::getName) + .collect(Collectors.toList()); + // We don't grant the privileges in one SQL, because some privilege has been granted, it + // will cause the failure of the SQL. So we grant the privileges one by one. + for (String privilege : privileges) { + List sqls = + getGrantPrivilegeSQL( + privilege, + convertedObject.metadataObjectType().name(), + convertedObject.fullName(), + role.name()); + for (String sql : sqls) { + executeUpdateSQL(sql, "is already granted"); + } + } + } + } + } + + private void revokeObjectPrivileges(Role role, SecurableObject removeObject) { + List authObjects = + mappingProvider.translatePrivilege(removeObject); + for (AuthorizationSecurableObject authObject : authObjects) { + List convertedObjects = Lists.newArrayList(); + if (authObject.name().equals(JdbcSecurableObject.ALL)) { + convertedObjects.addAll(convertResourceAll(authObject)); + } else { + convertedObjects.add(authObject); + } + + for (AuthorizationSecurableObject convertedObject : convertedObjects) { + List privileges = + filterUnsupportedPrivileges(authObject.privileges()).stream() + .map(AuthorizationPrivilege::getName) + .collect(Collectors.toList()); + for (String privilege : privileges) { + // We don't revoke the privileges in one SQL, because some privilege has been revoked, it + // will cause the failure of the SQL. So we revoke the privileges one by one. + List sqls = + getRevokePrivilegeSQL( + privilege, + convertedObject.metadataObjectType().name(), + convertedObject.fullName(), + role.name()); + for (String sql : sqls) { + executeUpdateSQL(sql, "Cannot find privilege Privilege"); + } + } + } + } + } +} diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java new file mode 100644 index 00000000000..b13504fd2fd --- /dev/null +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.jdbc; + +import java.util.Map; + +/** The properties for JDBC authorization plugin. */ +public class JdbcAuthorizationProperties { + private static final String CONFIG_PREFIX = "authorization.jdbc."; + public static final String JDBC_PASSWORD = CONFIG_PREFIX + "password"; + public static final String JDBC_USERNAME = CONFIG_PREFIX + "username"; + public static final String JDBC_URL = CONFIG_PREFIX + "url"; + public static final String JDBC_DRIVER = CONFIG_PREFIX + "driver"; + + public static void validate(Map properties) { + String errorMsg = "%s is required"; + check(properties, JDBC_URL, errorMsg); + check(properties, JDBC_USERNAME, errorMsg); + check(properties, JDBC_PASSWORD, errorMsg); + check(properties, JDBC_DRIVER, errorMsg); + } + + private static void check(Map properties, String key, String errorMsg) { + if (!properties.containsKey(key) && properties.get(key) != null) { + throw new IllegalArgumentException(String.format(errorMsg, key)); + } + } +} diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java new file mode 100644 index 00000000000..f7171ff354a --- /dev/null +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.jdbc; + +import java.util.List; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.annotation.Unstable; +import org.apache.gravitino.authorization.Owner; + +/** Interface for SQL operations of the underlying access control system. */ +@Unstable +interface JdbcAuthorizationSQL { + + /** + * Get SQL statements for creating a user. + * + * @param username the username to create + * @return the SQL statement list to create a user + */ + List getCreateUserSQL(String username); + + /** + * Get SQL statements for creating a group. + * + * @param username the username to drop + * @return the SQL statement list to drop a user + */ + List getDropUserSQL(String username); + + /** + * Get SQL statements for creating a role. + * + * @param roleName the role name to create + * @return the SQL statement list to create a role + */ + List getCreateRoleSQL(String roleName); + + /** + * Get SQL statements for dropping a role. + * + * @param roleName the role name to drop + * @return the SQL statement list to drop a role + */ + List getDropRoleSQL(String roleName); + + /** + * Get SQL statements for granting privileges. + * + * @param privilege the privilege to grant + * @param objectType the object type in the database system + * @param objectName the object name in the database system + * @param roleName the role name to grant + * @return the sql statement list to grant privilege + */ + List getGrantPrivilegeSQL( + String privilege, String objectType, String objectName, String roleName); + + /** + * Get SQL statements for revoking privileges. + * + * @param privilege the privilege to revoke + * @param objectType the object type in the database system + * @param objectName the object name in the database system + * @param roleName the role name to revoke + * @return the sql statement list to revoke privilege + */ + List getRevokePrivilegeSQL( + String privilege, String objectType, String objectName, String roleName); + + /** + * Get SQL statements for granting role. + * + * @param roleName the role name to grant + * @param grantorType the grantor type, usually USER or ROLE + * @param grantorName the grantor name + * @return the sql statement list to grant role + */ + List getGrantRoleSQL(String roleName, String grantorType, String grantorName); + + /** + * Get SQL statements for revoking roles. + * + * @param roleName the role name to revoke + * @param revokerType the revoker type, usually USER or ROLE + * @param revokerName the revoker name + * @return the sql statement list to revoke role + */ + List getRevokeRoleSQL(String roleName, String revokerType, String revokerName); + + /** + * Get SQL statements for setting owner. + * + * @param type The metadata object type + * @param objectName the object name in the database system + * @param preOwner the previous owner of the object + * @param newOwner the new owner of the object + * @return the sql statement list to set owner + */ + List getSetOwnerSQL( + MetadataObject.Type type, String objectName, Owner preOwner, Owner newOwner); +} diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java new file mode 100644 index 00000000000..c74c7ae6093 --- /dev/null +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.jdbc; + +import com.google.common.base.Preconditions; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.AuthorizationMetadataObject; + +public class JdbcMetadataObject implements AuthorizationMetadataObject { + + private final String parent; + private final String name; + private final Type type; + + public JdbcMetadataObject(String parent, String name, Type type) { + this.parent = parent; + this.name = name; + this.type = type; + } + + @Nullable + @Override + public String parent() { + return parent; + } + + @Override + public String name() { + return name; + } + + @Override + public List names() { + return DOT_SPLITTER.splitToList(fullName()); + } + + @Override + public Type type() { + return type; + } + + @Override + public void validateAuthorizationMetadataObject() throws IllegalArgumentException { + List names = names(); + Preconditions.checkArgument( + names != null && !names.isEmpty(), "The name of the object is empty."); + Preconditions.checkArgument( + names.size() <= 2, "The name of the object is not in the format of 'database.table'."); + Preconditions.checkArgument(type != null, "The type of the object is null."); + if (names.size() == 1) { + Preconditions.checkArgument( + type.metadataObjectType() == MetadataObject.Type.SCHEMA, + "The type of the object is not SCHEMA."); + } else { + Preconditions.checkArgument( + type.metadataObjectType() == MetadataObject.Type.TABLE, + "The type of the object is not TABLE."); + } + + for (String name : names) { + Preconditions.checkArgument(name != null, "Cannot create a metadata object with null name"); + } + } + + public enum Type implements AuthorizationMetadataObject.Type { + SCHEMA(MetadataObject.Type.SCHEMA), + TABLE(MetadataObject.Type.TABLE); + + private final MetadataObject.Type metadataType; + + Type(MetadataObject.Type type) { + this.metadataType = type; + } + + public MetadataObject.Type metadataObjectType() { + return metadataType; + } + + public static Type fromMetadataType(MetadataObject.Type metadataType) { + for (Type type : Type.values()) { + if (type.metadataObjectType() == metadataType) { + return type; + } + } + throw new IllegalArgumentException("No matching JdbcMetadataObject.Type for " + metadataType); + } + } +} diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java new file mode 100644 index 00000000000..845b31a5b59 --- /dev/null +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.jdbc; + +import org.apache.gravitino.authorization.AuthorizationPrivilege; +import org.apache.gravitino.authorization.Privilege; + +public enum JdbcPrivilege implements AuthorizationPrivilege { + SELECT("SELECT"), + INSERT("INSERT"), + UPDATE("UPDATE"), + ALTER("ALTER"), + DELETE("DELETE"), + ALL("ALL PRIVILEGES"), + CREATE("CREATE"), + DROP("DROP"), + USAGE("USAGE"); + + private final String name; + + JdbcPrivilege(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public Privilege.Condition condition() { + return Privilege.Condition.ALLOW; + } + + @Override + public boolean equalsTo(String value) { + return name.equals(value); + } +} diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java new file mode 100644 index 00000000000..78b82e2a8da --- /dev/null +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.jdbc; + +import java.util.List; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.AuthorizationPrivilege; +import org.apache.gravitino.authorization.AuthorizationSecurableObject; + +/** + * JdbcAuthorizationObject is used for translating securable object to authorization securable + * object. JdbcAuthorizationObject has the database and table name. When table name is null, the + * object represents a database. The database can't be null. + */ +public class JdbcSecurableObject extends JdbcMetadataObject + implements AuthorizationSecurableObject { + + public static final String ALL = "*"; + + List privileges; + + private JdbcSecurableObject( + String parent, + String name, + JdbcMetadataObject.Type type, + List privileges) { + super(parent, name, type); + this.privileges = privileges; + } + + static JdbcSecurableObject create( + String schema, String table, List privileges) { + String parent = table == null ? null : schema; + String name = table == null ? schema : table; + JdbcMetadataObject.Type type = + table == null + ? JdbcMetadataObject.Type.fromMetadataType(MetadataObject.Type.SCHEMA) + : JdbcMetadataObject.Type.fromMetadataType(MetadataObject.Type.TABLE); + + JdbcSecurableObject object = new JdbcSecurableObject(parent, name, type, privileges); + object.validateAuthorizationMetadataObject(); + return object; + } + + @Override + public List privileges() { + return privileges; + } +} diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java new file mode 100644 index 00000000000..70b2d10e39c --- /dev/null +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.jdbc; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.authorization.AuthorizationMetadataObject; +import org.apache.gravitino.authorization.AuthorizationPrivilege; +import org.apache.gravitino.authorization.AuthorizationPrivilegesMappingProvider; +import org.apache.gravitino.authorization.AuthorizationSecurableObject; +import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.authorization.SecurableObject; + +/** + * JdbcSecurableObjectMappingProvider is used for translating securable object to authorization + * securable object. + */ +public class JdbcSecurableObjectMappingProvider implements AuthorizationPrivilegesMappingProvider { + + private final Map> privilegeMapping = + ImmutableMap.of( + Privilege.Name.CREATE_TABLE, Sets.newHashSet(JdbcPrivilege.CREATE), + Privilege.Name.CREATE_SCHEMA, Sets.newHashSet(JdbcPrivilege.CREATE), + Privilege.Name.SELECT_TABLE, Sets.newHashSet(JdbcPrivilege.SELECT), + Privilege.Name.MODIFY_TABLE, + Sets.newHashSet( + JdbcPrivilege.SELECT, + JdbcPrivilege.UPDATE, + JdbcPrivilege.DELETE, + JdbcPrivilege.INSERT, + JdbcPrivilege.ALTER), + Privilege.Name.USE_SCHEMA, Sets.newHashSet(JdbcPrivilege.USAGE)); + + private final Map privilegeScopeMapping = + ImmutableMap.of( + Privilege.Name.CREATE_TABLE, MetadataObject.Type.TABLE, + Privilege.Name.CREATE_SCHEMA, MetadataObject.Type.SCHEMA, + Privilege.Name.SELECT_TABLE, MetadataObject.Type.TABLE, + Privilege.Name.MODIFY_TABLE, MetadataObject.Type.TABLE, + Privilege.Name.USE_SCHEMA, MetadataObject.Type.SCHEMA); + + private final Set ownerPrivileges = ImmutableSet.of(); + + private final Set allowObjectTypes = + ImmutableSet.of( + MetadataObject.Type.METALAKE, + MetadataObject.Type.CATALOG, + MetadataObject.Type.SCHEMA, + MetadataObject.Type.TABLE); + + @Override + public Map> privilegesMappingRule() { + return privilegeMapping; + } + + @Override + public Set ownerMappingRule() { + return ownerPrivileges; + } + + @Override + public Set allowPrivilegesRule() { + return privilegeMapping.keySet(); + } + + @Override + public Set allowMetadataObjectTypesRule() { + return allowObjectTypes; + } + + @Override + public List translatePrivilege(SecurableObject securableObject) { + List authObjects = Lists.newArrayList(); + List databasePrivileges = Lists.newArrayList(); + List tablePrivileges = Lists.newArrayList(); + JdbcSecurableObject databaseObject; + JdbcSecurableObject tableObject; + switch (securableObject.type()) { + case METALAKE: + case CATALOG: + convertJdbcPrivileges(securableObject, databasePrivileges, tablePrivileges); + + if (!databasePrivileges.isEmpty()) { + databaseObject = + JdbcSecurableObject.create(JdbcSecurableObject.ALL, null, databasePrivileges); + authObjects.add(databaseObject); + } + + if (!tablePrivileges.isEmpty()) { + tableObject = + JdbcSecurableObject.create( + JdbcSecurableObject.ALL, JdbcSecurableObject.ALL, tablePrivileges); + authObjects.add(tableObject); + } + break; + + case SCHEMA: + convertJdbcPrivileges(securableObject, databasePrivileges, tablePrivileges); + if (!databasePrivileges.isEmpty()) { + databaseObject = + JdbcSecurableObject.create(securableObject.name(), null, databasePrivileges); + authObjects.add(databaseObject); + } + + if (!tablePrivileges.isEmpty()) { + tableObject = + JdbcSecurableObject.create( + securableObject.name(), JdbcSecurableObject.ALL, tablePrivileges); + authObjects.add(tableObject); + } + break; + + case TABLE: + convertJdbcPrivileges(securableObject, databasePrivileges, tablePrivileges); + if (!tablePrivileges.isEmpty()) { + MetadataObject metadataObject = + MetadataObjects.parse(securableObject.parent(), MetadataObject.Type.SCHEMA); + tableObject = + JdbcSecurableObject.create( + metadataObject.name(), securableObject.name(), tablePrivileges); + authObjects.add(tableObject); + } + break; + + default: + throw new IllegalArgumentException( + String.format("Don't support metadata object type %s", securableObject.type())); + } + + return authObjects; + } + + @Override + public List translateOwner(MetadataObject metadataObject) { + List objects = Lists.newArrayList(); + switch (metadataObject.type()) { + case METALAKE: + case CATALOG: + objects.add( + JdbcSecurableObject.create( + JdbcSecurableObject.ALL, null, Lists.newArrayList(JdbcPrivilege.ALL))); + objects.add( + JdbcSecurableObject.create( + JdbcSecurableObject.ALL, + JdbcSecurableObject.ALL, + Lists.newArrayList(JdbcPrivilege.ALL))); + break; + case SCHEMA: + objects.add( + JdbcSecurableObject.create( + metadataObject.name(), null, Lists.newArrayList(JdbcPrivilege.ALL))); + objects.add( + JdbcSecurableObject.create( + metadataObject.name(), + JdbcSecurableObject.ALL, + Lists.newArrayList(JdbcPrivilege.ALL))); + break; + case TABLE: + MetadataObject schema = + MetadataObjects.parse(metadataObject.parent(), MetadataObject.Type.SCHEMA); + objects.add( + JdbcSecurableObject.create( + schema.name(), metadataObject.name(), Lists.newArrayList(JdbcPrivilege.ALL))); + break; + default: + throw new IllegalArgumentException( + "Don't support metadata object type " + metadataObject.type()); + } + return objects; + } + + @Override + public AuthorizationMetadataObject translateMetadataObject(MetadataObject metadataObject) { + throw new UnsupportedOperationException("Not supported"); + } + + private void convertJdbcPrivileges( + SecurableObject securableObject, + List databasePrivileges, + List tablePrivileges) { + for (Privilege privilege : securableObject.privileges()) { + if (privilegeScopeMapping.get(privilege.name()) == MetadataObject.Type.SCHEMA) { + databasePrivileges.addAll(privilegeMapping.get(privilege.name())); + } else if (privilegeScopeMapping.get(privilege.name()) == MetadataObject.Type.TABLE) { + tablePrivileges.addAll(privilegeMapping.get(privilege.name())); + } + } + } +} diff --git a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java b/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java new file mode 100644 index 00000000000..b72392a6cd8 --- /dev/null +++ b/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.jdbc; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.Owner; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.RoleChange; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.User; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.GroupEntity; +import org.apache.gravitino.meta.RoleEntity; +import org.apache.gravitino.meta.UserEntity; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class JdbcAuthorizationPluginTest { + private static List expectSQLs = Lists.newArrayList(); + private static List expectTypes = Lists.newArrayList(); + private static List expectObjectNames = Lists.newArrayList(); + private static List> expectPreOwners = Lists.newArrayList(); + private static List expectNewOwners = Lists.newArrayList(); + private static int currentSQLIndex = 0; + private static int currentIndex = 0; + private static final Map properties = + ImmutableMap.of( + JdbcAuthorizationProperties.JDBC_URL, + "xx", + JdbcAuthorizationProperties.JDBC_USERNAME, + "xx", + JdbcAuthorizationProperties.JDBC_PASSWORD, + "xx", + JdbcAuthorizationProperties.JDBC_DRIVER, + "xx"); + + private static final JdbcAuthorizationPlugin plugin = + new JdbcAuthorizationPlugin(properties) { + + @Override + public List getSetOwnerSQL( + MetadataObject.Type type, String objectName, Owner preOwner, Owner newOwner) { + Assertions.assertEquals(expectTypes.get(currentIndex), type); + Assertions.assertEquals(expectObjectNames.get(currentIndex), objectName); + Assertions.assertEquals(expectPreOwners.get(currentIndex), Optional.ofNullable(preOwner)); + Assertions.assertEquals(expectNewOwners.get(currentIndex), newOwner); + currentIndex++; + return Collections.emptyList(); + } + + void executeUpdateSQL(String sql, String ignoreErrorMsg) { + Assertions.assertEquals(expectSQLs.get(currentSQLIndex), sql); + currentSQLIndex++; + } + }; + + @Test + public void testUserManagement() { + expectSQLs = Lists.newArrayList("CREATE USER tmp"); + currentSQLIndex = 0; + plugin.onUserAdded(createUser("tmp")); + + Assertions.assertThrows( + UnsupportedOperationException.class, () -> plugin.onUserAcquired(createUser("tmp"))); + + expectSQLs = Lists.newArrayList("DROP USER tmp"); + currentSQLIndex = 0; + plugin.onUserRemoved(createUser("tmp")); + } + + @Test + public void testGroupManagement() { + expectSQLs = Lists.newArrayList("CREATE USER GRAVITINO_GROUP_tmp"); + resetSQLIndex(); + plugin.onGroupAdded(createGroup("tmp")); + + Assertions.assertThrows( + UnsupportedOperationException.class, () -> plugin.onGroupAcquired(createGroup("tmp"))); + + expectSQLs = Lists.newArrayList("DROP USER GRAVITINO_GROUP_tmp"); + resetSQLIndex(); + plugin.onGroupRemoved(createGroup("tmp")); + } + + @Test + public void testRoleManagement() { + expectSQLs = Lists.newArrayList("CREATE ROLE tmp"); + resetSQLIndex(); + Role role = createRole("tmp"); + plugin.onRoleCreated(role); + + Assertions.assertThrows(UnsupportedOperationException.class, () -> plugin.onRoleAcquired(role)); + + resetSQLIndex(); + expectSQLs = Lists.newArrayList("DROP ROLE tmp"); + plugin.onRoleDeleted(role); + } + + @Test + public void testPermissionManagement() { + Role role = createRole("tmp"); + Group group = createGroup("tmp"); + User user = createUser("tmp"); + + resetSQLIndex(); + expectSQLs = + Lists.newArrayList("CREATE ROLE tmp", "GRANT ROLE tmp TO USER GRAVITINO_GROUP_tmp"); + plugin.onGrantedRolesToGroup(Lists.newArrayList(role), group); + + resetSQLIndex(); + expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "GRANT ROLE tmp TO USER tmp"); + plugin.onGrantedRolesToUser(Lists.newArrayList(role), user); + + resetSQLIndex(); + expectSQLs = + Lists.newArrayList("CREATE ROLE tmp", "REVOKE ROLE tmp FROM USER GRAVITINO_GROUP_tmp"); + plugin.onRevokedRolesFromGroup(Lists.newArrayList(role), group); + + resetSQLIndex(); + expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "REVOKE ROLE tmp FROM USER tmp"); + plugin.onRevokedRolesFromUser(Lists.newArrayList(role), user); + + // Test metalake object and different role change + resetSQLIndex(); + expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE *.* TO ROLE tmp"); + SecurableObject metalakeObject = + SecurableObjects.ofMetalake("metalake", Lists.newArrayList(Privileges.SelectTable.allow())); + RoleChange roleChange = RoleChange.addSecurableObject("tmp", metalakeObject); + plugin.onRoleUpdated(role, roleChange); + + resetSQLIndex(); + expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "REVOKE SELECT ON TABLE *.* FROM ROLE tmp"); + roleChange = RoleChange.removeSecurableObject("tmp", metalakeObject); + plugin.onRoleUpdated(role, roleChange); + + resetSQLIndex(); + expectSQLs = + Lists.newArrayList( + "CREATE ROLE tmp", + "REVOKE SELECT ON TABLE *.* FROM ROLE tmp", + "GRANT CREATE ON TABLE *.* TO ROLE tmp"); + SecurableObject newMetalakeObject = + SecurableObjects.ofMetalake("metalake", Lists.newArrayList(Privileges.CreateTable.allow())); + roleChange = RoleChange.updateSecurableObject("tmp", metalakeObject, newMetalakeObject); + plugin.onRoleUpdated(role, roleChange); + + // Test catalog object + resetSQLIndex(); + SecurableObject catalogObject = + SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.SelectTable.allow())); + roleChange = RoleChange.addSecurableObject("tmp", catalogObject); + expectSQLs = Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE *.* TO ROLE tmp"); + plugin.onRoleUpdated(role, roleChange); + + // Test schema object + resetSQLIndex(); + SecurableObject schemaObject = + SecurableObjects.ofSchema( + catalogObject, "schema", Lists.newArrayList(Privileges.SelectTable.allow())); + roleChange = RoleChange.addSecurableObject("tmp", schemaObject); + expectSQLs = + Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE schema.* TO ROLE tmp"); + plugin.onRoleUpdated(role, roleChange); + + // Test table object + resetSQLIndex(); + SecurableObject tableObject = + SecurableObjects.ofTable( + schemaObject, "table", Lists.newArrayList(Privileges.SelectTable.allow())); + roleChange = RoleChange.addSecurableObject("tmp", tableObject); + expectSQLs = + Lists.newArrayList("CREATE ROLE tmp", "GRANT SELECT ON TABLE schema.table TO ROLE tmp"); + plugin.onRoleUpdated(role, roleChange); + } + + @Test + public void testOwnerManagement() { + + // Test metalake object + Owner owner = new TemporaryOwner("tmp", Owner.Type.USER); + MetadataObject metalakeObject = + MetadataObjects.of(null, "metalake", MetadataObject.Type.METALAKE); + expectSQLs = Lists.newArrayList("CREATE USER tmp"); + currentSQLIndex = 0; + expectTypes.add(MetadataObject.Type.SCHEMA); + expectObjectNames.add("*"); + expectPreOwners.add(Optional.empty()); + expectNewOwners.add(owner); + + expectTypes.add(MetadataObject.Type.TABLE); + expectObjectNames.add("*.*"); + expectPreOwners.add(Optional.empty()); + expectNewOwners.add(owner); + plugin.onOwnerSet(metalakeObject, null, owner); + + // clean up + cleanup(); + expectSQLs = Lists.newArrayList("CREATE USER tmp"); + + // Test catalog object + MetadataObject catalogObject = MetadataObjects.of(null, "catalog", MetadataObject.Type.CATALOG); + expectTypes.add(MetadataObject.Type.SCHEMA); + expectObjectNames.add("*"); + expectPreOwners.add(Optional.empty()); + expectNewOwners.add(owner); + + expectTypes.add(MetadataObject.Type.TABLE); + expectObjectNames.add("*.*"); + expectPreOwners.add(Optional.empty()); + expectNewOwners.add(owner); + plugin.onOwnerSet(catalogObject, null, owner); + + // clean up + cleanup(); + expectSQLs = Lists.newArrayList("CREATE USER tmp"); + + // Test schema object + MetadataObject schemaObject = + MetadataObjects.of("catalog", "schema", MetadataObject.Type.SCHEMA); + expectTypes.add(MetadataObject.Type.SCHEMA); + expectObjectNames.add("schema"); + expectPreOwners.add(Optional.empty()); + expectNewOwners.add(owner); + + expectTypes.add(MetadataObject.Type.TABLE); + expectObjectNames.add("schema.*"); + expectPreOwners.add(Optional.empty()); + expectNewOwners.add(owner); + plugin.onOwnerSet(schemaObject, null, owner); + + // clean up + cleanup(); + expectSQLs = Lists.newArrayList("CREATE USER tmp"); + + // Test table object + MetadataObject tableObject = + MetadataObjects.of( + Lists.newArrayList("catalog", "schema", "table"), MetadataObject.Type.TABLE); + + expectTypes.add(MetadataObject.Type.TABLE); + expectObjectNames.add("schema.table"); + expectPreOwners.add(Optional.empty()); + expectNewOwners.add(owner); + plugin.onOwnerSet(tableObject, null, owner); + } + + private static void resetSQLIndex() { + currentSQLIndex = 0; + } + + private static void cleanup() { + expectTypes.clear(); + expectObjectNames.clear(); + expectPreOwners.clear(); + expectNewOwners.clear(); + currentIndex = 0; + currentSQLIndex = 0; + } + + private static class TemporaryOwner implements Owner { + private final String name; + private final Type type; + + public TemporaryOwner(String name, Type type) { + this.name = name; + this.type = type; + } + + @Override + public String name() { + return name; + } + + @Override + public Type type() { + return type; + } + } + + private static Role createRole(String name) { + return RoleEntity.builder().withId(0L).withName(name).withAuditInfo(AuditInfo.EMPTY).build(); + } + + private static Group createGroup(String name) { + return GroupEntity.builder().withId(0L).withName(name).withAuditInfo(AuditInfo.EMPTY).build(); + } + + private static User createUser(String name) { + return UserEntity.builder().withId(0L).withName(name).withAuditInfo(AuditInfo.EMPTY).build(); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 150acdb00ce..b3eb56578aa 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,7 +57,7 @@ if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() == true) } include("iceberg:iceberg-common") include("iceberg:iceberg-rest-server") -include("authorizations:authorization-ranger") +include("authorizations:authorization-ranger", "authorizations:authorization-jdbc") include("trino-connector:trino-connector", "trino-connector:integration-test") include("spark-connector:spark-common") // kyuubi hive connector doesn't support 2.13 for Spark3.3 From f6806bb9f76c881fee3879197a053b84740e1eab Mon Sep 17 00:00:00 2001 From: FANNG Date: Tue, 24 Dec 2024 16:24:03 +0800 Subject: [PATCH 29/47] [#5954] feat(iceberg): Supports ADLS for Iceberg catalog and spark connector (#5952) ### What changes were proposed in this pull request? 1. Most code work is implemented in #5938 #5737 including catalog properties convert and add Iceberg azure bundle jar, this PR mainly about test and document. 2. Remove hidden properties of the cloud secret key from the Iceberg catalog, as Gravitino doesn't have an unified security management yet and Iceberg REST server need to fetch catalog cloud properties to initiate `IcebergWrapper` dymaticly. Another benefit is spark connector does not need to specify the secret key explictly. Supports ADLS for Iceberg catalog and spark connector ### Why are the changes needed? Fix: #5954 ### Does this PR introduce _any_ user-facing change? Yes, the user no need to specify the cloud secret key in spark connector. ### How was this patch tested? test in local enviroment --- LICENSE.bin | 1 + .../IcebergCatalogPropertiesMetadata.java | 21 +++++++++++++++---- docs/lakehouse-iceberg-catalog.md | 21 +++++++++++++++---- docs/spark-connector/spark-catalog-iceberg.md | 18 +++++++++++++--- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/LICENSE.bin b/LICENSE.bin index 34723024e78..effaa4ac4a2 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -304,6 +304,7 @@ Apache Iceberg Aliyun Apache Iceberg api Apache Iceberg AWS + Apache Iceberg Azure Apache Iceberg core Apache Iceberg Hive metastore Apache Iceberg GCP diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java index 375edd600fb..9e1c184cad9 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java @@ -33,6 +33,7 @@ import org.apache.gravitino.iceberg.common.IcebergCatalogBackend; import org.apache.gravitino.iceberg.common.authentication.AuthenticationConfig; import org.apache.gravitino.iceberg.common.authentication.kerberos.KerberosConfig; +import org.apache.gravitino.storage.AzureProperties; import org.apache.gravitino.storage.OSSProperties; import org.apache.gravitino.storage.S3Properties; @@ -91,25 +92,37 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad "s3 access key ID", false /* immutable */, null /* defaultValue */, - true /* hidden */), + false /* hidden */), stringOptionalPropertyEntry( S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, "s3 secret access key", false /* immutable */, null /* defaultValue */, - true /* hidden */), + false /* hidden */), stringOptionalPropertyEntry( OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, "OSS access key ID", false /* immutable */, null /* defaultValue */, - true /* hidden */), + false /* hidden */), stringOptionalPropertyEntry( OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, "OSS access key secret", false /* immutable */, null /* defaultValue */, - true /* hidden */)); + false /* hidden */), + stringOptionalPropertyEntry( + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, + "Azure storage account name", + false /* immutable */, + null /* defaultValue */, + false /* hidden */), + stringOptionalPropertyEntry( + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, + "Azure storage account key", + false /* immutable */, + null /* defaultValue */, + false /* hidden */)); HashMap> result = Maps.newHashMap(); result.putAll(Maps.uniqueIndex(propertyEntries, PropertyEntry::getName)); result.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES); diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 393ef26b8cf..6ad011d7160 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -28,10 +28,7 @@ Builds with Apache Iceberg `1.5.2`. The Apache Iceberg table format version is ` - Works as a catalog proxy, supporting `Hive`, `JDBC` and `REST` as catalog backend. - Supports DDL operations for Iceberg schemas and tables. - Doesn't support snapshot or table management operations. -- Supports multi storage. - - S3 - - HDFS - - OSS +- Supports multi storage, including S3, GCS, ADLS, OSS and HDFS. - Supports Kerberos or simple authentication for Iceberg catalog with Hive backend. ### Catalog properties @@ -119,6 +116,22 @@ Please make sure the credential file is accessible by Gravitino, like using `exp Please set `warehouse` to `gs://{bucket_name}/${prefix_name}`, and download [Iceberg GCP bundle jar](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-gcp-bundle) and place it to `catalogs/lakehouse-iceberg/libs/`. ::: +#### ADLS + +Supports using Azure account name and secret key to access ADLS data. + +| Configuration item | Description | Default value | Required | Since Version | +|------------------------------|-----------------------------------------------------------------------------------------------------------|---------------|----------|------------------| +| `io-impl` | The io implementation for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for ADLS. | (none) | No | 0.6.0-incubating | +| `azure-storage-account-name` | The static storage account name used to access ADLS data. | (none) | No | 0.8.0-incubating | +| `azure-storage-account-key` | The static storage account key used to access ADLS data. | (none) | No | 0.8.0-incubating | + +For other Iceberg ADLS properties not managed by Gravitino like `adls.read.block-size-bytes`, you could config it directly by `gravitino.iceberg-rest.adls.read.block-size-bytes`. + +:::info +Please set `warehouse` to `abfs[s]://{container-name}@{storage-account-name}.dfs.core.windows.net/{path}`, and download the [Iceberg Azure bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-azure-bundle) and place it to `catalogs/lakehouse-iceberg/libs/`. +::: + #### Other storages For other storages that are not managed by Gravitino directly, you can manage them through custom catalog properties. diff --git a/docs/spark-connector/spark-catalog-iceberg.md b/docs/spark-connector/spark-catalog-iceberg.md index e4933a3036f..28f2b55c7e6 100644 --- a/docs/spark-connector/spark-catalog-iceberg.md +++ b/docs/spark-connector/spark-catalog-iceberg.md @@ -111,7 +111,13 @@ Gravitino spark connector will transform below property names which are defined | `io-impl` | `io-impl` | The io implementation for `FileIO` in Iceberg. | 0.6.0-incubating | | `s3-endpoint` | `s3.endpoint` | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | 0.6.0-incubating | | `s3-region` | `client.region` | The region of the S3 service, like `us-west-2`. | 0.6.0-incubating | +| `s3-access-key-id` | `s3.access-key-id` | The static access key ID used to access S3 data. | 0.8.0-incubating | +| `s3-secret-access-key` | `s3.secret-access-key` | The static secret access key used to access S3 data. | 0.8.0-incubating | | `oss-endpoint` | `oss.endpoint` | The endpoint of Aliyun OSS service. | 0.7.0-incubating | +| `oss-access-key-id` | `client.access-key-id` | The static access key ID used to access OSS data. | 0.8.0-incubating | +| `oss-secret-access-key` | `client.access-key-secret` | The static secret access key used to access OSS data. | 0.8.0-incubating | +| `azure-storage-account-name` | `adls.auth.shared-key.account.name` | The static storage account name used to access ADLS data. | 0.8.0-incubating | +| `azure-storage-account-key` | `adls.auth.shared-key.account.key` | The static storage account key used to access ADLS data.. | 0.8.0-incubating | Gravitino catalog property names with the prefix `spark.bypass.` are passed to Spark Iceberg connector. For example, using `spark.bypass.clients` to pass the `clients` to the Spark Iceberg connector. @@ -121,17 +127,23 @@ Iceberg catalog property `cache-enabled` is setting to `false` internally and no ## Storage +Spark connector could convert storage properties in the Gravitino catalog to Spark Iceberg connector automatically, No extra configuration is needed for `S3`, `ADLS`, `OSS`, `GCS`. + ### S3 -You need to add s3 secret to the Spark configuration using `spark.sql.catalog.${iceberg_catalog_name}.s3.access-key-id` and `spark.sql.catalog.${iceberg_catalog_name}.s3.secret-access-key`. Additionally, download the [Iceberg AWS bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-aws-bundle) and place it in the classpath of Spark. +Please downloading the [Iceberg AWS bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-aws-bundle) and place it in the classpath of Spark. ### OSS -You need to add OSS secret key to the Spark configuration using `spark.sql.catalog.${iceberg_catalog_name}.client.access-key-id` and `spark.sql.catalog.${iceberg_catalog_name}.client.access-key-secret`. Additionally, download the [Aliyun OSS SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in the classpath of Spark. +Please downloading the [Aliyun OSS SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in the classpath of Spark. ### GCS -No extra configuration is needed. Please make sure the credential file is accessible by Spark, like using `export GOOGLE_APPLICATION_CREDENTIALS=/xx/application_default_credentials.json`, and download [Iceberg GCP bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-gcp-bundle) and place it to the classpath of Spark. +Please make sure the credential file is accessible by Spark, like using `export GOOGLE_APPLICATION_CREDENTIALS=/xx/application_default_credentials.json`, and download [Iceberg GCP bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-gcp-bundle) and place it to the classpath of Spark. + +### ADLS + +Please downloading the [Iceberg Azure bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-azure-bundle) and place it in the classpath of Spark. ### Other storage From 798d5e7cbacaee76ca67c25701c27144f7d929e1 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Tue, 24 Dec 2024 19:03:43 +0800 Subject: [PATCH 30/47] Minor (docs): Update how-to-use-the-playground.md document (#5955) ### What changes were proposed in this pull request? Revert helm-chart related descriptions in `docs/how-to-use-the-playground.md`. ### Why are the changes needed? Make documentation match `README.md` of `apache/gravitino-playground` . ### Does this PR introduce _any_ user-facing change? Yes, documentation is affacted. ### How was this patch tested? Manually. --- docs/how-to-use-the-playground.md | 73 +++---------------------------- 1 file changed, 6 insertions(+), 67 deletions(-) diff --git a/docs/how-to-use-the-playground.md b/docs/how-to-use-the-playground.md index 390e7a37be4..d65d40acdd2 100644 --- a/docs/how-to-use-the-playground.md +++ b/docs/how-to-use-the-playground.md @@ -14,7 +14,6 @@ Depending on your network and computer, startup time may take 3-5 minutes. Once ## Prerequisites Install Git (optional), Docker, Docker Compose. -Docker Desktop (or Orbstack) with Kubernetes enabled and helm CLI is required if you use helm-chart to deploy services. ## System Resource Requirements @@ -50,82 +49,22 @@ git clone git@github.com:apache/gravitino-playground.git cd gravitino-playground ``` -#### Docker - -##### Start - -``` -./playground.sh docker start -``` - -##### Check status - -```shell -./playground.sh docker status -``` - -##### Stop playground - -```shell -./playground.sh docker stop -``` - -#### Kubernetes - -Enable Kubernetes in Docker Desktop or Orbstack. - -In the project root directory, execute this command: - -``` -helm upgrade --install gravitino-playground ./helm-chart/ --create-namespace --namespace gravitino-playground --set projectRoot=$(pwd) -``` - -##### Start +#### Start ``` -./playground.sh k8s start +./playground.sh start ``` -##### Check status +#### Check status ```shell -./playground.sh k8s status -``` - -##### Port Forwarding - -To access pods or services at `localhost`, you need to do these steps: - -1. Log in to the Gravitino playground Trino pod using the following command: - -``` -TRINO_POD=$(kubectl get pods --namespace gravitino-playground -l app=trino -o jsonpath="{.items[0].metadata.name}") -kubectl exec $TRINO_POD -n gravitino-playground -it -- /bin/bash -``` - -2. Log in to the Gravitino playground Spark pod using the following command: - -``` -SPARK_POD=$(kubectl get pods --namespace gravitino-playground -l app=spark -o jsonpath="{.items[0].metadata.name}") -kubectl exec $SPARK_POD -n gravitino-playground -it -- /bin/bash -``` - -3. Port-forward the Gravitino service to access it at `localhost:8090`. - -``` -kubectl port-forward svc/gravitino -n gravitino-playground 8090:8090 -``` - -4. Port-forward the Jupyter Notebook service to access it at `localhost:8888`. - -``` -kubectl port-forward svc/jupyternotebook -n gravitino-playground 8888:8888 +./playground.sh status ``` -##### Stop playground +#### Stop playground ```shell -./playground.sh k8s stop +./playground.sh stop ``` ## Experiencing Apache Gravitino with Trino SQL From 13cd1a928cdecf9919fe645effb67ccd540cc42b Mon Sep 17 00:00:00 2001 From: Xun Date: Wed, 25 Dec 2024 15:38:24 +0800 Subject: [PATCH 31/47] [#5775] feat(auth): Chain authorization plugin framework (#5786) ### What changes were proposed in this pull request? 1. Add Chain auth plugin module 1. Add auth common module 3. Add Chain authorization Ranger Hive and Ranger HDFS ITs ### Why are the changes needed? Fix: #5775 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Add ITs --- .../access-control-integration-test.yml | 9 +- .../authorization-chain/build.gradle.kts | 146 +++++++ .../chain/ChainedAuthorization.java | 17 +- .../chain/ChainedAuthorizationPlugin.java | 197 +++++++++ ...nector.authorization.AuthorizationProvider | 19 + .../test/TestChainedAuthorizationIT.java | 374 ++++++++++++++++++ .../src/test/resources/log4j2.properties | 73 ++++ .../ranger-spark-security.xml.template | 45 +++ .../authorization-common/build.gradle.kts | 62 +++ .../common/AuthorizationProperties.java | 54 +++ .../ChainedAuthorizationProperties.java} | 36 +- .../common}/JdbcAuthorizationProperties.java | 28 +- .../common/PathBasedMetadataObject.java} | 33 +- .../common/PathBasedSecurableObject.java} | 6 +- .../RangerAuthorizationProperties.java | 16 +- .../TestChainedAuthorizationProperties.java} | 84 +++- .../TestRangerAuthorizationProperties.java | 51 ++- .../authorization-jdbc/build.gradle.kts | 4 +- .../jdbc/JdbcAuthorizationPlugin.java | 4 +- .../jdbc/JdbcAuthorizationPluginTest.java | 1 + .../authorization-ranger/build.gradle.kts | 27 +- .../ranger/RangerAuthorization.java | 10 +- .../ranger/RangerAuthorizationHDFSPlugin.java | 195 +++++++-- .../RangerAuthorizationHadoopSQLPlugin.java | 44 +-- .../ranger/RangerAuthorizationPlugin.java | 5 +- .../ranger/RangerHadoopSQLMetadataObject.java | 2 +- .../authorization/ranger/RangerHelper.java | 2 +- .../ranger/RangerPrivileges.java | 26 ++ .../test/RangerAuthorizationHDFSPluginIT.java | 15 +- .../integration/test/RangerBaseE2EIT.java | 11 +- .../integration/test/RangerFilesetIT.java | 2 +- .../integration/test/RangerHiveE2EIT.java | 12 +- .../ranger/integration/test/RangerITEnv.java | 11 +- .../integration/test/RangerIcebergE2EIT.java | 12 +- .../integration/test/RangerPaimonE2EIT.java | 12 +- authorizations/build.gradle.kts | 18 +- build.gradle.kts | 8 +- .../integration/test/ProxyCatalogHiveIT.java | 18 - .../gravitino/catalog/CatalogManager.java | 35 +- .../gravitino/connector/BaseCatalog.java | 35 +- .../authorization/BaseAuthorization.java | 72 ++++ .../authorization/TestAuthorization.java | 62 +-- .../ranger/TestRangerAuthorization.java | 18 +- .../TestRangerAuthorizationHDFSPlugin.java} | 4 +- ...stRangerAuthorizationHadoopSQLPlugin.java} | 2 +- ...nector.authorization.AuthorizationProvider | 3 +- integration-test-common/build.gradle.kts | 5 +- .../integration/test/util/BaseIT.java | 38 ++ settings.gradle.kts | 2 +- 49 files changed, 1630 insertions(+), 335 deletions(-) create mode 100644 authorizations/authorization-chain/build.gradle.kts rename core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java => authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorization.java (71%) create mode 100644 authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java create mode 100644 authorizations/authorization-chain/src/main/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider create mode 100644 authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java create mode 100644 authorizations/authorization-chain/src/test/resources/log4j2.properties create mode 100644 authorizations/authorization-chain/src/test/resources/ranger-spark-security.xml.template create mode 100644 authorizations/authorization-common/build.gradle.kts create mode 100644 authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java rename authorizations/{authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/ChainAuthorizationProperties.java => authorization-common/src/main/java/org/apache/gravitino/authorization/common/ChainedAuthorizationProperties.java} (87%) rename authorizations/{authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc => authorization-common/src/main/java/org/apache/gravitino/authorization/common}/JdbcAuthorizationProperties.java (73%) rename authorizations/{authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java => authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java} (64%) rename authorizations/{authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java => authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedSecurableObject.java} (89%) rename authorizations/{authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger => authorization-common/src/main/java/org/apache/gravitino/authorization/common}/RangerAuthorizationProperties.java (90%) rename authorizations/{authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestChainAuthorizationProperties.java => authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestChainedAuthorizationProperties.java} (75%) rename authorizations/{authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger => authorization-common/src/test/java/org/apache/gravitino/authorization/common}/TestRangerAuthorizationProperties.java (72%) rename core/src/test/java/org/apache/gravitino/connector/authorization/{mysql/TestMySQLAuthorizationPlugin.java => ranger/TestRangerAuthorizationHDFSPlugin.java} (95%) rename core/src/test/java/org/apache/gravitino/connector/authorization/ranger/{TestRangerAuthorizationPlugin.java => TestRangerAuthorizationHadoopSQLPlugin.java} (97%) diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index 6997eaf9a4c..dc8acd60678 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -87,12 +87,9 @@ jobs: - name: Authorization Integration Test (JDK${{ matrix.java-version }}) id: integrationTest run: | - ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test - ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test - ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test - ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test - ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test - ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test + ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test + ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test + ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/authorizations/authorization-chain/build.gradle.kts b/authorizations/authorization-chain/build.gradle.kts new file mode 100644 index 00000000000..d5cd160742c --- /dev/null +++ b/authorizations/authorization-chain/build.gradle.kts @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "authorization-chain" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() +val sparkVersion: String = libs.versions.spark35.get() +val kyuubiVersion: String = libs.versions.kyuubi4paimon.get() +val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".") + +dependencies { + implementation(project(":api")) { + exclude(group = "*") + } + implementation(project(":core")) { + exclude(group = "*") + } + implementation(project(":common")) { + exclude(group = "*") + } + implementation(project(":authorizations:authorization-common")) { + exclude(group = "*") + } + implementation(libs.bundles.log4j) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.javax.jaxb.api) { + exclude("*") + } + implementation(libs.javax.ws.rs.api) + implementation(libs.jettison) + implementation(libs.rome) + compileOnly(libs.lombok) + + testImplementation(project(":core")) + testImplementation(project(":clients:client-java")) + testImplementation(project(":server")) + testImplementation(project(":catalogs:catalog-common")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":authorizations:authorization-ranger")) + testImplementation(project(":authorizations:authorization-ranger", "testArtifacts")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.mockito.core) + testImplementation(libs.testcontainers) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) + testImplementation(libs.ranger.intg) { + exclude("org.apache.hadoop", "hadoop-common") + exclude("org.apache.hive", "hive-storage-api") + exclude("org.apache.lucene") + exclude("org.apache.solr") + exclude("org.apache.kafka") + exclude("org.elasticsearch") + exclude("org.elasticsearch.client") + exclude("org.elasticsearch.plugin") + exclude("org.apache.ranger", "ranger-plugins-audit") + exclude("org.apache.ranger", "ranger-plugins-cred") + exclude("org.apache.ranger", "ranger-plugin-classloader") + exclude("net.java.dev.jna") + exclude("javax.ws.rs") + exclude("org.eclipse.jetty") + } + testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") + testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { + exclude("org.apache.avro") + exclude("org.apache.hadoop") + exclude("org.apache.zookeeper") + exclude("io.dropwizard.metrics") + exclude("org.rocksdb") + } + testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") { + exclude("com.sun.jersey") + } + testImplementation(libs.hadoop3.client) + testImplementation(libs.hadoop3.common) { + exclude("com.sun.jersey") + exclude("javax.servlet", "servlet-api") + } + testImplementation(libs.hadoop3.hdfs) { + exclude("com.sun.jersey") + exclude("javax.servlet", "servlet-api") + exclude("io.netty") + } +} + +tasks { + val runtimeJars by registering(Copy::class) { + from(configurations.runtimeClasspath) + into("build/libs") + } + + val copyAuthorizationLibs by registering(Copy::class) { + dependsOn("jar", runtimeJars) + from("build/libs") { + exclude("guava-*.jar") + exclude("log4j-*.jar") + exclude("slf4j-*.jar") + } + into("$rootDir/distribution/package/authorizations/chain/libs") + } + + register("copyLibAndConfig", Copy::class) { + dependsOn(copyAuthorizationLibs) + } + + jar { + dependsOn(runtimeJars) + } +} + +tasks.test { + doFirst { + environment("HADOOP_USER_NAME", "gravitino") + } + dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars", ":authorizations:authorization-ranger:jar", ":authorizations:authorization-ranger:runtimeJars") + + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/test/**") + } else { + dependsOn(tasks.jar) + } +} diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java b/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorization.java similarity index 71% rename from core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java rename to authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorization.java index e8d747da11f..5f8c9834750 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java +++ b/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorization.java @@ -16,24 +16,27 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.connector.authorization.mysql; +package org.apache.gravitino.authorization.chain; import java.util.Map; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; -public class TestMySQLAuthorization extends BaseAuthorization { - - public TestMySQLAuthorization() {} - +/** Implementation of a Chained authorization in Gravitino. */ +public class ChainedAuthorization extends BaseAuthorization { @Override public String shortName() { - return "mysql"; + return "chain"; } @Override public AuthorizationPlugin newPlugin( String metalake, String catalogProvider, Map config) { - return new TestMySQLAuthorizationPlugin(); + switch (catalogProvider) { + case "hive": + return new ChainedAuthorizationPlugin(metalake, catalogProvider, config); + default: + throw new IllegalArgumentException("Unknown catalog provider: " + catalogProvider); + } } } diff --git a/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java b/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java new file mode 100644 index 00000000000..120c355db06 --- /dev/null +++ b/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.chain; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.MetadataObjectChange; +import org.apache.gravitino.authorization.Owner; +import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.RoleChange; +import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.common.AuthorizationProperties; +import org.apache.gravitino.authorization.common.ChainedAuthorizationProperties; +import org.apache.gravitino.connector.authorization.AuthorizationPlugin; +import org.apache.gravitino.connector.authorization.BaseAuthorization; +import org.apache.gravitino.exceptions.AuthorizationPluginException; +import org.apache.gravitino.utils.IsolatedClassLoader; + +/** Chained authorization operations plugin class.
*/ +public class ChainedAuthorizationPlugin implements AuthorizationPlugin { + private List plugins = Lists.newArrayList(); + private final String metalake; + + public ChainedAuthorizationPlugin( + String metalake, String catalogProvider, Map config) { + this.metalake = metalake; + initPlugins(catalogProvider, config); + } + + private void initPlugins(String catalogProvider, Map properties) { + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + chainedAuthzProperties.validate(); + // Validate the properties for each plugin + chainedAuthzProperties + .plugins() + .forEach( + pluginName -> { + Map pluginProperties = + chainedAuthzProperties.fetchAuthPluginProperties(pluginName); + String authzProvider = chainedAuthzProperties.getPluginProvider(pluginName); + AuthorizationProperties.validate(authzProvider, pluginProperties); + }); + // Create the plugins + chainedAuthzProperties + .plugins() + .forEach( + pluginName -> { + String authzProvider = chainedAuthzProperties.getPluginProvider(pluginName); + Map pluginConfig = + chainedAuthzProperties.fetchAuthPluginProperties(pluginName); + + ArrayList libAndResourcesPaths = Lists.newArrayList(); + BaseAuthorization.buildAuthorizationPkgPath( + ImmutableMap.of(Catalog.AUTHORIZATION_PROVIDER, authzProvider)) + .ifPresent(libAndResourcesPaths::add); + IsolatedClassLoader classLoader = + IsolatedClassLoader.buildClassLoader(libAndResourcesPaths); + try { + BaseAuthorization authorization = + BaseAuthorization.createAuthorization(classLoader, authzProvider); + AuthorizationPlugin authorizationPlugin = + authorization.newPlugin(metalake, catalogProvider, pluginConfig); + plugins.add(authorizationPlugin); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public void close() throws IOException { + for (AuthorizationPlugin plugin : plugins) { + plugin.close(); + } + } + + @Override + public Boolean onMetadataUpdated(MetadataObjectChange... changes) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onMetadataUpdated(changes)); + } + + @Override + public Boolean onRoleCreated(Role role) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRoleCreated(role)); + } + + @Override + public Boolean onRoleAcquired(Role role) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRoleAcquired(role)); + } + + @Override + public Boolean onRoleDeleted(Role role) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRoleDeleted(role)); + } + + @Override + public Boolean onRoleUpdated(Role role, RoleChange... changes) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRoleUpdated(role, changes)); + } + + @Override + public Boolean onGrantedRolesToUser(List roles, User user) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGrantedRolesToUser(roles, user)); + } + + @Override + public Boolean onRevokedRolesFromUser(List roles, User user) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRevokedRolesFromUser(roles, user)); + } + + @Override + public Boolean onGrantedRolesToGroup(List roles, Group group) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGrantedRolesToGroup(roles, group)); + } + + @Override + public Boolean onRevokedRolesFromGroup(List roles, Group group) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRevokedRolesFromGroup(roles, group)); + } + + @Override + public Boolean onUserAdded(User user) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onUserAdded(user)); + } + + @Override + public Boolean onUserRemoved(User user) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onUserRemoved(user)); + } + + @Override + public Boolean onUserAcquired(User user) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onUserAcquired(user)); + } + + @Override + public Boolean onGroupAdded(Group group) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGroupAdded(group)); + } + + @Override + public Boolean onGroupRemoved(Group group) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGroupRemoved(group)); + } + + @Override + public Boolean onGroupAcquired(Group group) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGroupAcquired(group)); + } + + @Override + public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onOwnerSet(metadataObject, preOwner, newOwner)); + } + + private Boolean chainedAction(Function action) { + for (AuthorizationPlugin plugin : plugins) { + if (!action.apply(plugin)) { + return false; + } + } + return true; + } +} diff --git a/authorizations/authorization-chain/src/main/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider b/authorizations/authorization-chain/src/main/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider new file mode 100644 index 00000000000..f4bea1086db --- /dev/null +++ b/authorizations/authorization-chain/src/main/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.authorization.chain.ChainedAuthorization \ No newline at end of file diff --git a/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java new file mode 100644 index 00000000000..74ad99aa7f9 --- /dev/null +++ b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.chain.integration.test; + +import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; +import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.Configs; +import org.apache.gravitino.auth.AuthConstants; +import org.apache.gravitino.auth.AuthenticatorType; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.common.ChainedAuthorizationProperties; +import org.apache.gravitino.authorization.ranger.integration.test.RangerBaseE2EIT; +import org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv; +import org.apache.gravitino.catalog.hive.HiveConstants; +import org.apache.gravitino.exceptions.UserAlreadyExistsException; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.kyuubi.plugin.spark.authz.AccessControlException; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestChainedAuthorizationIT extends RangerBaseE2EIT { + private static final Logger LOG = LoggerFactory.getLogger(TestChainedAuthorizationIT.class); + private static String DEFAULT_FS; + private FileSystem fileSystem; + + @BeforeAll + public void startIntegrationTest() throws Exception { + metalakeName = GravitinoITUtils.genRandomName("metalake").toLowerCase(); + // Enable Gravitino Authorization mode + Map configs = Maps.newHashMap(); + configs.put(Configs.ENABLE_AUTHORIZATION.getKey(), String.valueOf(true)); + configs.put(Configs.SERVICE_ADMINS.getKey(), RangerITEnv.HADOOP_USER_NAME); + configs.put(Configs.AUTHENTICATORS.getKey(), AuthenticatorType.SIMPLE.name().toLowerCase()); + configs.put("SimpleAuthUserName", AuthConstants.ANONYMOUS_USER); + registerCustomConfigs(configs); + + super.startIntegrationTest(); + RangerITEnv.init(RangerBaseE2EIT.metalakeName, false); + RangerITEnv.startHiveRangerContainer(); + + HIVE_METASTORE_URIS = + String.format( + "thrift://%s:%d", + containerSuite.getHiveRangerContainer().getContainerIpAddress(), + HiveContainer.HIVE_METASTORE_PORT); + + generateRangerSparkSecurityXML("authorization-chain"); + + DEFAULT_FS = + String.format( + "hdfs://%s:%d/user/hive/warehouse", + containerSuite.getHiveRangerContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT); + BaseIT.runInEnv( + "HADOOP_USER_NAME", + AuthConstants.ANONYMOUS_USER, + () -> { + sparkSession = + SparkSession.builder() + .master("local[1]") + .appName("Ranger Hive E2E integration test") + .config("hive.metastore.uris", HIVE_METASTORE_URIS) + .config("spark.sql.warehouse.dir", DEFAULT_FS) + .config("spark.sql.storeAssignmentPolicy", "LEGACY") + .config("mapreduce.input.fileinputformat.input.dir.recursive", "true") + .config( + "spark.sql.extensions", + "org.apache.kyuubi.plugin.spark.authz.ranger.RangerSparkExtension") + .enableHiveSupport() + .getOrCreate(); + sparkSession.sql(SQL_SHOW_DATABASES); // must be called to activate the Spark session + }); + createMetalake(); + createCatalog(); + + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", DEFAULT_FS); + fileSystem = FileSystem.get(conf); + + RangerITEnv.cleanup(); + try { + metalake.addUser(System.getenv(HADOOP_USER_NAME)); + } catch (UserAlreadyExistsException e) { + LOG.error("Failed to add user: {}", System.getenv(HADOOP_USER_NAME), e); + } + } + + @AfterAll + public void stop() throws IOException { + if (client != null) { + Arrays.stream(catalog.asSchemas().listSchemas()) + .filter(schema -> !schema.equals("default")) + .forEach( + (schema -> { + catalog.asSchemas().dropSchema(schema, false); + })); + Arrays.stream(metalake.listCatalogs()) + .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); + client.disableMetalake(metalakeName); + client.dropMetalake(metalakeName); + } + if (fileSystem != null) { + fileSystem.close(); + } + try { + closer.close(); + } catch (Exception e) { + LOG.error("Failed to close CloseableGroup", e); + } + client = null; + RangerITEnv.cleanup(); + } + + private String storageLocation(String dirName) { + return DEFAULT_FS + "/" + dirName; + } + + @Test + public void testCreateSchemaInCatalog() throws IOException { + // Choose a catalog + useCatalog(); + + // First, fail to create the schema + Exception accessControlException = + Assertions.assertThrows(Exception.class, () -> sparkSession.sql(SQL_CREATE_SCHEMA)); + Assertions.assertTrue( + accessControlException + .getMessage() + .contains( + String.format( + "Permission denied: user [%s] does not have [create] privilege", + AuthConstants.ANONYMOUS_USER)) + || accessControlException + .getMessage() + .contains( + String.format( + "Permission denied: user=%s, access=WRITE", AuthConstants.ANONYMOUS_USER))); + Path schemaPath = new Path(storageLocation(schemaName + ".db")); + Assertions.assertFalse(fileSystem.exists(schemaPath)); + FileStatus fileStatus = fileSystem.getFileStatus(new Path(DEFAULT_FS)); + Assertions.assertEquals(System.getenv(HADOOP_USER_NAME), fileStatus.getOwner()); + + // Second, grant the `CREATE_SCHEMA` role + String roleName = currentFunName(); + SecurableObject securableObject = + SecurableObjects.ofCatalog( + catalogName, Lists.newArrayList(Privileges.CreateSchema.allow())); + metalake.createRole(roleName, Collections.emptyMap(), Lists.newArrayList(securableObject)); + metalake.grantRolesToUser(Lists.newArrayList(roleName), AuthConstants.ANONYMOUS_USER); + waitForUpdatingPolicies(); + + // Third, succeed to create the schema + sparkSession.sql(SQL_CREATE_SCHEMA); + Assertions.assertTrue(fileSystem.exists(schemaPath)); + FileStatus fsSchema = fileSystem.getFileStatus(schemaPath); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, fsSchema.getOwner()); + + // Fourth, fail to create the table + Assertions.assertThrows(AccessControlException.class, () -> sparkSession.sql(SQL_CREATE_TABLE)); + + // Clean up + catalog.asSchemas().dropSchema(schemaName, false); + metalake.deleteRole(roleName); + waitForUpdatingPolicies(); + + Exception accessControlException2 = + Assertions.assertThrows(Exception.class, () -> sparkSession.sql(SQL_CREATE_SCHEMA)); + Assertions.assertTrue( + accessControlException2 + .getMessage() + .contains( + String.format( + "Permission denied: user [%s] does not have [create] privilege", + AuthConstants.ANONYMOUS_USER)) + || accessControlException2 + .getMessage() + .contains( + String.format( + "Permission denied: user=%s, access=WRITE", AuthConstants.ANONYMOUS_USER))); + } + + @Override + public void createCatalog() { + Map catalogConf = new HashMap<>(); + catalogConf.put(HiveConstants.METASTORE_URIS, HIVE_METASTORE_URIS); + catalogConf.put(IMPERSONATION_ENABLE, "true"); + catalogConf.put(Catalog.AUTHORIZATION_PROVIDER, "chain"); + catalogConf.put(ChainedAuthorizationProperties.CHAIN_PLUGINS_PROPERTIES_KEY, "hive1,hdfs1"); + catalogConf.put("authorization.chain.hive1.provider", "ranger"); + catalogConf.put("authorization.chain.hive1.ranger.auth.type", RangerContainer.authType); + catalogConf.put("authorization.chain.hive1.ranger.admin.url", RangerITEnv.RANGER_ADMIN_URL); + catalogConf.put("authorization.chain.hive1.ranger.username", RangerContainer.rangerUserName); + catalogConf.put("authorization.chain.hive1.ranger.password", RangerContainer.rangerPassword); + catalogConf.put("authorization.chain.hive1.ranger.service.type", "HadoopSQL"); + catalogConf.put( + "authorization.chain.hive1.ranger.service.name", RangerITEnv.RANGER_HIVE_REPO_NAME); + catalogConf.put("authorization.chain.hdfs1.provider", "ranger"); + catalogConf.put("authorization.chain.hdfs1.ranger.auth.type", RangerContainer.authType); + catalogConf.put("authorization.chain.hdfs1.ranger.admin.url", RangerITEnv.RANGER_ADMIN_URL); + catalogConf.put("authorization.chain.hdfs1.ranger.username", RangerContainer.rangerUserName); + catalogConf.put("authorization.chain.hdfs1.ranger.password", RangerContainer.rangerPassword); + catalogConf.put("authorization.chain.hdfs1.ranger.service.type", "HDFS"); + catalogConf.put( + "authorization.chain.hdfs1.ranger.service.name", RangerITEnv.RANGER_HDFS_REPO_NAME); + + metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, "hive", "comment", catalogConf); + catalog = metalake.loadCatalog(catalogName); + LOG.info("Catalog created: {}", catalog); + } + + @Test + public void testCreateSchema() throws InterruptedException { + // TODO + } + + @Test + void testCreateTable() throws InterruptedException { + // TODO + } + + @Test + void testReadWriteTableWithMetalakeLevelRole() throws InterruptedException { + // TODO + } + + @Test + void testReadWriteTableWithTableLevelRole() throws InterruptedException { + // TODO + } + + @Test + void testReadOnlyTable() throws InterruptedException { + // TODO + } + + @Test + void testWriteOnlyTable() throws InterruptedException { + // TODO + } + + @Test + void testCreateAllPrivilegesRole() throws InterruptedException { + // TODO + } + + @Test + void testDeleteAndRecreateRole() throws InterruptedException { + // TODO + } + + @Test + void testDeleteAndRecreateMetadataObject() throws InterruptedException { + // TODO + } + + @Test + void testRenameMetadataObject() throws InterruptedException { + // TODO + } + + @Test + void testRenameMetadataObjectPrivilege() throws InterruptedException { + // TODO + } + + @Test + void testChangeOwner() throws InterruptedException { + // TODO + } + + @Test + void testAllowUseSchemaPrivilege() throws InterruptedException { + // TODO + } + + @Test + void testDenyPrivileges() throws InterruptedException { + // TODO + } + + @Test + void testGrantPrivilegesForMetalake() throws InterruptedException { + // TODO + } + + @Override + protected void checkTableAllPrivilegesExceptForCreating() { + // TODO + } + + @Override + protected void checkUpdateSQLWithReadWritePrivileges() { + // TODO + } + + @Override + protected void checkUpdateSQLWithReadPrivileges() { + // TODO + } + + @Override + protected void checkUpdateSQLWithWritePrivileges() { + // TODO + } + + @Override + protected void checkDeleteSQLWithReadWritePrivileges() { + // TODO + } + + @Override + protected void checkDeleteSQLWithReadPrivileges() { + // TODO + } + + @Override + protected void checkDeleteSQLWithWritePrivileges() { + // TODO + } + + @Override + protected void useCatalog() { + // TODO + } + + @Override + protected void checkWithoutPrivileges() { + // TODO + } + + @Override + protected void testAlterTable() { + // TODO + } +} diff --git a/authorizations/authorization-chain/src/test/resources/log4j2.properties b/authorizations/authorization-chain/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..2a46c57ec2f --- /dev/null +++ b/authorizations/authorization-chain/src/test/resources/log4j2.properties @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-build/authorization-chain-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger + +# File appender configuration for testcontainers +appender.testcontainersFile.type = File +appender.testcontainersFile.name = testcontainersLogger +appender.testcontainersFile.fileName = build/testcontainers.log +appender.testcontainersFile.layout.type = PatternLayout +appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Logger for testcontainers +logger.testcontainers.name = org.testcontainers +logger.testcontainers.level = debug +logger.testcontainers.additivity = false +logger.testcontainers.appenderRef.file.ref = testcontainersLogger + +logger.tc.name = tc +logger.tc.level = debug +logger.tc.additivity = false +logger.tc.appenderRef.file.ref = testcontainersLogger + +logger.docker.name = com.github.dockerjava +logger.docker.level = warn +logger.docker.additivity = false +logger.docker.appenderRef.file.ref = testcontainersLogger + +logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire +logger.http.level = off diff --git a/authorizations/authorization-chain/src/test/resources/ranger-spark-security.xml.template b/authorizations/authorization-chain/src/test/resources/ranger-spark-security.xml.template new file mode 100644 index 00000000000..eb7f2b5e811 --- /dev/null +++ b/authorizations/authorization-chain/src/test/resources/ranger-spark-security.xml.template @@ -0,0 +1,45 @@ + + + + ranger.plugin.spark.policy.rest.url + __REPLACE__RANGER_ADMIN_URL + + + + ranger.plugin.spark.service.name + __REPLACE__RANGER_HIVE_REPO_NAME + + + + ranger.plugin.spark.policy.cache.dir + /tmp/policycache + + + + ranger.plugin.spark.policy.pollIntervalMs + 500 + + + + ranger.plugin.spark.policy.source.impl + org.apache.ranger.admin.client.RangerAdminRESTClient + + + \ No newline at end of file diff --git a/authorizations/authorization-common/build.gradle.kts b/authorizations/authorization-common/build.gradle.kts new file mode 100644 index 00000000000..ba64510f2ce --- /dev/null +++ b/authorizations/authorization-common/build.gradle.kts @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "authorization-chain" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +dependencies { + implementation(project(":api")) { + exclude(group = "*") + } + implementation(project(":core")) { + exclude(group = "*") + } + implementation(project(":common")) { + exclude(group = "*") + } + implementation(libs.bundles.log4j) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.javax.jaxb.api) { + exclude("*") + } + implementation(libs.javax.ws.rs.api) + implementation(libs.jettison) + implementation(libs.rome) + compileOnly(libs.lombok) + + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.mockito.core) + testImplementation(libs.testcontainers) + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks.test { + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/test/**") + } else { + dependsOn(tasks.jar) + } +} diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java new file mode 100644 index 00000000000..3005cc5f3e9 --- /dev/null +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.common; + +import java.util.Map; +import java.util.stream.Collectors; + +public abstract class AuthorizationProperties { + protected Map properties; + + public AuthorizationProperties(Map properties) { + this.properties = + properties.entrySet().stream() + .filter(entry -> entry.getKey().startsWith(getPropertiesPrefix())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + abstract String getPropertiesPrefix(); + + abstract void validate(); + + public static void validate(String type, Map properties) { + switch (type) { + case "ranger": + RangerAuthorizationProperties rangerAuthorizationProperties = + new RangerAuthorizationProperties(properties); + rangerAuthorizationProperties.validate(); + break; + case "chain": + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + chainedAuthzProperties.validate(); + break; + default: + throw new IllegalArgumentException("Unsupported authorization properties type: " + type); + } + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/ChainAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/ChainedAuthorizationProperties.java similarity index 87% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/ChainAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/ChainedAuthorizationProperties.java index edaa375747a..7e5aea0ca29 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/ChainAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/ChainedAuthorizationProperties.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -29,7 +29,7 @@ import java.util.stream.Collectors; /** - * The properties for Chain authorization plugin.
+ * The properties for Chained authorization plugin.
*
* Configuration Example:
* "authorization.chain.plugins" = "hive1,hdfs1"
@@ -48,16 +48,32 @@ * "authorization.chain.hdfs1.ranger.username" = "admin";
* "authorization.chain.hdfs1.ranger.password" = "admin";
*/ -public class ChainAuthorizationProperties { - public static final String PLUGINS_SPLITTER = ","; - /** Chain authorization plugin names */ +public class ChainedAuthorizationProperties extends AuthorizationProperties { + private static final String PLUGINS_SPLITTER = ","; + /** Chained authorization plugin names */ public static final String CHAIN_PLUGINS_PROPERTIES_KEY = "authorization.chain.plugins"; - /** Chain authorization plugin provider */ + /** Chained authorization plugin provider */ public static final String CHAIN_PROVIDER = "authorization.chain.*.provider"; - static Map fetchAuthPluginProperties( - String pluginName, Map properties) { + public ChainedAuthorizationProperties(Map properties) { + super(properties); + } + + @Override + public String getPropertiesPrefix() { + return "authorization.chain"; + } + + public String getPluginProvider(String pluginName) { + return properties.get(getPropertiesPrefix() + "." + pluginName + ".provider"); + } + + public List plugins() { + return Arrays.asList(properties.get(CHAIN_PLUGINS_PROPERTIES_KEY).split(PLUGINS_SPLITTER)); + } + + public Map fetchAuthPluginProperties(String pluginName) { Preconditions.checkArgument( properties.containsKey(CHAIN_PLUGINS_PROPERTIES_KEY) && properties.get(CHAIN_PLUGINS_PROPERTIES_KEY) != null, @@ -93,13 +109,15 @@ static Map fetchAuthPluginProperties( return resultProperties; } - public static void validate(Map properties) { + @Override + public void validate() { Preconditions.checkArgument( properties.containsKey(CHAIN_PLUGINS_PROPERTIES_KEY), String.format("%s is required", CHAIN_PLUGINS_PROPERTIES_KEY)); List pluginNames = Arrays.stream(properties.get(CHAIN_PLUGINS_PROPERTIES_KEY).split(PLUGINS_SPLITTER)) .map(String::trim) + .filter(v -> !v.isEmpty()) .collect(Collectors.toList()); Preconditions.checkArgument( !pluginNames.isEmpty(), diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java similarity index 73% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java index b13504fd2fd..9a5e7c6cc97 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java @@ -16,29 +16,39 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.jdbc; +package org.apache.gravitino.authorization.common; import java.util.Map; /** The properties for JDBC authorization plugin. */ -public class JdbcAuthorizationProperties { +public class JdbcAuthorizationProperties extends AuthorizationProperties { private static final String CONFIG_PREFIX = "authorization.jdbc."; public static final String JDBC_PASSWORD = CONFIG_PREFIX + "password"; public static final String JDBC_USERNAME = CONFIG_PREFIX + "username"; public static final String JDBC_URL = CONFIG_PREFIX + "url"; public static final String JDBC_DRIVER = CONFIG_PREFIX + "driver"; - public static void validate(Map properties) { - String errorMsg = "%s is required"; - check(properties, JDBC_URL, errorMsg); - check(properties, JDBC_USERNAME, errorMsg); - check(properties, JDBC_PASSWORD, errorMsg); - check(properties, JDBC_DRIVER, errorMsg); + public JdbcAuthorizationProperties(Map properties) { + super(properties); } - private static void check(Map properties, String key, String errorMsg) { + private void check(String key, String errorMsg) { if (!properties.containsKey(key) && properties.get(key) != null) { throw new IllegalArgumentException(String.format(errorMsg, key)); } } + + @Override + String getPropertiesPrefix() { + return CONFIG_PREFIX; + } + + @Override + public void validate() { + String errorMsg = "%s is required"; + check(JDBC_URL, errorMsg); + check(JDBC_USERNAME, errorMsg); + check(JDBC_PASSWORD, errorMsg); + check(JDBC_DRIVER, errorMsg); + } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java similarity index 64% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java index 77523464162..ed67b1cc0fc 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -25,10 +25,10 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.AuthorizationMetadataObject; -public class RangerPathBaseMetadataObject implements AuthorizationMetadataObject { +public class PathBasedMetadataObject implements AuthorizationMetadataObject { /** - * The type of object in the Ranger system. Every type will map one kind of the entity of the - * Gravitino type system. + * The type of metadata object in the underlying system. Every type will map one kind of the + * entity of the Gravitino type system. */ public enum Type implements AuthorizationMetadataObject.Type { /** A path is mapped the path of storages like HDFS, S3 etc. */ @@ -42,24 +42,13 @@ public enum Type implements AuthorizationMetadataObject.Type { public MetadataObject.Type metadataObjectType() { return metadataType; } - - public static RangerHadoopSQLMetadataObject.Type fromMetadataType( - MetadataObject.Type metadataType) { - for (RangerHadoopSQLMetadataObject.Type type : RangerHadoopSQLMetadataObject.Type.values()) { - if (type.metadataObjectType() == metadataType) { - return type; - } - } - throw new IllegalArgumentException( - "No matching RangerMetadataObject.Type for " + metadataType); - } } private final String path; private final AuthorizationMetadataObject.Type type; - public RangerPathBaseMetadataObject(String path, AuthorizationMetadataObject.Type type) { + public PathBasedMetadataObject(String path, AuthorizationMetadataObject.Type type) { this.path = path; this.type = type; } @@ -89,18 +78,20 @@ public AuthorizationMetadataObject.Type type() { public void validateAuthorizationMetadataObject() throws IllegalArgumentException { List names = names(); Preconditions.checkArgument( - names != null && !names.isEmpty(), "Cannot create a Ranger metadata object with no names"); + names != null && !names.isEmpty(), + "Cannot create a path based metadata object with no names"); Preconditions.checkArgument( names.size() == 1, - "Cannot create a Ranger metadata object with the name length which is 1"); + "Cannot create a path based metadata object with the name length which is 1"); Preconditions.checkArgument( - type != null, "Cannot create a Ranger metadata object with no type"); + type != null, "Cannot create a path based metadata object with no type"); Preconditions.checkArgument( - type == RangerPathBaseMetadataObject.Type.PATH, "it must be the PATH type"); + type == PathBasedMetadataObject.Type.PATH, "it must be the PATH type"); for (String name : names) { - Preconditions.checkArgument(name != null, "Cannot create a metadata object with null name"); + Preconditions.checkArgument( + name != null, "Cannot create a path based metadata object with null name"); } } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedSecurableObject.java similarity index 89% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedSecurableObject.java index bd2c73fdaef..6712cdf0e3d 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedSecurableObject.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.collect.ImmutableList; import java.util.List; @@ -25,12 +25,12 @@ import org.apache.gravitino.authorization.AuthorizationPrivilege; import org.apache.gravitino.authorization.AuthorizationSecurableObject; -public class RangerPathBaseSecurableObject extends RangerPathBaseMetadataObject +public class PathBasedSecurableObject extends PathBasedMetadataObject implements AuthorizationSecurableObject { private final List privileges; - public RangerPathBaseSecurableObject( + public PathBasedSecurableObject( String path, AuthorizationMetadataObject.Type type, Set privileges) { super(path, type); this.privileges = ImmutableList.copyOf(privileges); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/RangerAuthorizationProperties.java similarity index 90% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/RangerAuthorizationProperties.java index e7fee3088f6..73af3bc377e 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/RangerAuthorizationProperties.java @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.base.Preconditions; import java.util.Map; /** The properties for Ranger authorization plugin. */ -public class RangerAuthorizationProperties { +public class RangerAuthorizationProperties extends AuthorizationProperties { /** Ranger admin web URIs */ public static final String RANGER_ADMIN_URL = "authorization.ranger.admin.url"; @@ -46,7 +46,17 @@ public class RangerAuthorizationProperties { */ public static final String RANGER_PASSWORD = "authorization.ranger.password"; - public static void validate(Map properties) { + public RangerAuthorizationProperties(Map properties) { + super(properties); + } + + @Override + public String getPropertiesPrefix() { + return "authorization.ranger"; + } + + @Override + public void validate() { Preconditions.checkArgument( properties.containsKey(RANGER_ADMIN_URL), String.format("%s is required", RANGER_ADMIN_URL)); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestChainAuthorizationProperties.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestChainedAuthorizationProperties.java similarity index 75% rename from authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestChainAuthorizationProperties.java rename to authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestChainedAuthorizationProperties.java index 5d19f234093..7c0cccc738c 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestChainAuthorizationProperties.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestChainedAuthorizationProperties.java @@ -16,21 +16,22 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; -import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; import com.google.common.collect.Maps; import java.util.HashMap; import java.util.Map; -import org.apache.gravitino.catalog.hive.HiveConstants; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class TestChainAuthorizationProperties { +public class TestChainedAuthorizationProperties { + static final String METASTORE_URIS = "metastore.uris"; + public static final String IMPERSONATION_ENABLE = "impersonation-enable"; + @Test - void testChainOnePlugin() { + void testChainedOnePlugin() { Map properties = Maps.newHashMap(); properties.put("authorization.chain.plugins", "hive1"); properties.put("authorization.chain.hive1.provider", "ranger"); @@ -40,13 +41,15 @@ void testChainOnePlugin() { properties.put("authorization.chain.hive1.ranger.password", "admin"); properties.put("authorization.chain.hive1.ranger.service.type", "hive"); properties.put("authorization.chain.hive1.ranger.service.name", "hiveDev"); - Assertions.assertDoesNotThrow(() -> ChainAuthorizationProperties.validate(properties)); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + Assertions.assertDoesNotThrow(() -> chainedAuthzProperties.validate()); } @Test - void testChainTwoPlugins() { + void testChainedTwoPlugins() { Map properties = new HashMap<>(); - properties.put(HiveConstants.METASTORE_URIS, "thrift://localhost:9083"); + properties.put(METASTORE_URIS, "thrift://localhost:9083"); properties.put("gravitino.bypass.hive.metastore.client.capability.check", "true"); properties.put(IMPERSONATION_ENABLE, "true"); properties.put(AUTHORIZATION_PROVIDER, "chain"); @@ -65,7 +68,26 @@ void testChainTwoPlugins() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); - Assertions.assertDoesNotThrow(() -> ChainAuthorizationProperties.validate(properties)); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + Assertions.assertDoesNotThrow(() -> chainedAuthzProperties.validate()); + } + + @Test + void testWithoutPlugins() { + Map properties = Maps.newHashMap(); + properties.put("authorization.chain.plugins", ""); + properties.put("authorization.chain.hive1.provider", "ranger"); + properties.put("authorization.chain.hive1.ranger.auth.type", "simple"); + properties.put("authorization.chain.hive1.ranger.admin.url", "http://localhost:6080"); + properties.put("authorization.chain.hive1.ranger.username", "admin"); + properties.put("authorization.chain.hive1.ranger.password", "admin"); + properties.put("authorization.chain.hive1.ranger.service.type", "hive"); + properties.put("authorization.chain.hive1.ranger.service.name", "hiveDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + Assertions.assertThrows( + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test @@ -86,7 +108,9 @@ void testPluginsHasSpace() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); - Assertions.assertDoesNotThrow(() -> ChainAuthorizationProperties.validate(properties)); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + Assertions.assertDoesNotThrow(() -> chainedAuthzProperties.validate()); } @Test @@ -107,8 +131,10 @@ void testPluginsOneButHasTowPluginConfig() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); Assertions.assertThrows( - IllegalArgumentException.class, () -> ChainAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test @@ -129,8 +155,10 @@ void testPluginsHasPoint() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); Assertions.assertThrows( - IllegalArgumentException.class, () -> ChainAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test @@ -151,8 +179,10 @@ void testErrorPluginName() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.plug3.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); Assertions.assertThrows( - IllegalArgumentException.class, () -> ChainAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test @@ -173,14 +203,16 @@ void testDuplicationPluginName() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); Assertions.assertThrows( - IllegalArgumentException.class, () -> ChainAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test void testFetchRangerPrpoerties() { Map properties = new HashMap<>(); - properties.put(HiveConstants.METASTORE_URIS, "thrift://localhost:9083"); + properties.put(METASTORE_URIS, "thrift://localhost:9083"); properties.put("gravitino.bypass.hive.metastore.client.capability.check", "true"); properties.put(IMPERSONATION_ENABLE, "true"); properties.put(AUTHORIZATION_PROVIDER, "chain"); @@ -199,15 +231,25 @@ void testFetchRangerPrpoerties() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); - Map rangerHiveProperties = - ChainAuthorizationProperties.fetchAuthPluginProperties("hive1", properties); Assertions.assertDoesNotThrow( - () -> RangerAuthorizationProperties.validate(rangerHiveProperties)); + () -> { + Map rangerHiveProperties = + chainedAuthzProperties.fetchAuthPluginProperties("hive1"); + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(rangerHiveProperties); + rangerAuthProperties.validate(); + }); - Map rangerHDFSProperties = - ChainAuthorizationProperties.fetchAuthPluginProperties("hdfs1", properties); Assertions.assertDoesNotThrow( - () -> RangerAuthorizationProperties.validate(rangerHDFSProperties)); + () -> { + Map rangerHDFSProperties = + chainedAuthzProperties.fetchAuthPluginProperties("hdfs1"); + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(rangerHDFSProperties); + rangerAuthProperties.validate(); + }); } } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationProperties.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestRangerAuthorizationProperties.java similarity index 72% rename from authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationProperties.java rename to authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestRangerAuthorizationProperties.java index a90b164a21f..b2fcf6fc811 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationProperties.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestRangerAuthorizationProperties.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.collect.Maps; import java.util.Map; @@ -33,7 +33,12 @@ void testRangerProperties() { properties.put("authorization.ranger.password", "admin"); properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); - Assertions.assertDoesNotThrow(() -> RangerAuthorizationProperties.validate(properties)); + Assertions.assertDoesNotThrow( + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -45,7 +50,12 @@ void testRangerPropertiesLoseAuthType() { properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -57,7 +67,12 @@ void testRangerPropertiesLoseAdminUrl() { properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -69,7 +84,12 @@ void testRangerPropertiesLoseUserName() { properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -81,7 +101,12 @@ void testRangerPropertiesLosePassword() { properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -93,7 +118,12 @@ void testRangerPropertiesLoseServiceType() { properties.put("authorization.ranger.password", "admin"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -105,6 +135,11 @@ void testRangerPropertiesLoseServiceName() { properties.put("authorization.ranger.password", "admin"); properties.put("authorization.ranger.service.type", "hive"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } } diff --git a/authorizations/authorization-jdbc/build.gradle.kts b/authorizations/authorization-jdbc/build.gradle.kts index 8b105908c26..1a61f7c0cf9 100644 --- a/authorizations/authorization-jdbc/build.gradle.kts +++ b/authorizations/authorization-jdbc/build.gradle.kts @@ -31,7 +31,9 @@ dependencies { implementation(project(":core")) { exclude(group = "*") } - + implementation(project(":authorizations:authorization-common")) { + exclude(group = "*") + } implementation(libs.bundles.log4j) implementation(libs.commons.lang3) implementation(libs.guava) diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java index f889cee2240..d9bc28636c3 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java @@ -40,6 +40,7 @@ import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.meta.AuditInfo; @@ -65,7 +66,8 @@ abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAutho public JdbcAuthorizationPlugin(Map config) { // Initialize the data source dataSource = new BasicDataSource(); - JdbcAuthorizationProperties.validate(config); + JdbcAuthorizationProperties jdbcAuthProperties = new JdbcAuthorizationProperties(config); + jdbcAuthProperties.validate(); String jdbcUrl = config.get(JdbcAuthorizationProperties.JDBC_URL); dataSource.setUrl(jdbcUrl); diff --git a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java b/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java index b72392a6cd8..e261fad78d2 100644 --- a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java +++ b/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java @@ -34,6 +34,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; import org.apache.gravitino.meta.RoleEntity; diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts index a335e492b31..d410b1ee8d4 100644 --- a/authorizations/authorization-ranger/build.gradle.kts +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -38,7 +38,12 @@ dependencies { implementation(project(":core")) { exclude(group = "*") } - + implementation(project(":catalogs:catalog-common")) { + exclude(group = "*") + } + implementation(project(":authorizations:authorization-common")) { + exclude(group = "*") + } implementation(libs.bundles.log4j) implementation(libs.commons.lang3) implementation(libs.guava) @@ -47,10 +52,8 @@ dependencies { } implementation(libs.javax.ws.rs.api) implementation(libs.jettison) - compileOnly(libs.lombok) implementation(libs.mail) implementation(libs.ranger.intg) { - exclude("org.apache.hadoop", "hadoop-common") exclude("org.apache.hive", "hive-storage-api") exclude("org.apache.lucene") exclude("org.apache.solr") @@ -66,16 +69,15 @@ dependencies { exclude("org.eclipse.jetty") } implementation(libs.rome) - + compileOnly(libs.lombok) + testRuntimeOnly(libs.junit.jupiter.engine) testImplementation(project(":common")) testImplementation(project(":clients:client-java")) testImplementation(project(":server")) - testImplementation(project(":catalogs:catalog-common")) testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.testcontainers) - testRuntimeOnly(libs.junit.jupiter.engine) testImplementation(libs.mysql.driver) testImplementation(libs.postgresql.driver) testImplementation(libs.postgresql.driver) @@ -143,3 +145,16 @@ tasks.test { dependsOn(tasks.jar) } } + +val testJar by tasks.registering(Jar::class) { + archiveClassifier.set("tests") + from(sourceSets["test"].output) +} + +configurations { + create("testArtifacts") +} + +artifacts { + add("testArtifacts", testJar) +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java index 6aae714a359..b179b94c025 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java @@ -18,10 +18,9 @@ */ package org.apache.gravitino.authorization.ranger; -import static org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties.RANGER_SERVICE_TYPE; - import com.google.common.base.Preconditions; import java.util.Map; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; @@ -36,9 +35,10 @@ public String shortName() { public AuthorizationPlugin newPlugin( String metalake, String catalogProvider, Map properties) { Preconditions.checkArgument( - properties.containsKey(RANGER_SERVICE_TYPE), - String.format("%s is required", RANGER_SERVICE_TYPE)); - String serviceType = properties.get(RANGER_SERVICE_TYPE).toUpperCase(); + properties.containsKey(RangerAuthorizationProperties.RANGER_SERVICE_TYPE), + String.format("%s is required", RangerAuthorizationProperties.RANGER_SERVICE_TYPE)); + String serviceType = + properties.get(RangerAuthorizationProperties.RANGER_SERVICE_TYPE).toUpperCase(); switch (serviceType) { case "HADOOPSQL": return new RangerAuthorizationHadoopSQLPlugin(metalake, properties); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 9afa77880e9..bc3d309e1d1 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.authorization.ranger; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -30,26 +31,29 @@ import java.util.Objects; import java.util.Set; import java.util.regex.Pattern; +import org.apache.gravitino.Catalog; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Schema; import org.apache.gravitino.authorization.AuthorizationMetadataObject; import org.apache.gravitino.authorization.AuthorizationPrivilege; import org.apache.gravitino.authorization.AuthorizationSecurableObject; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.common.PathBasedMetadataObject; +import org.apache.gravitino.authorization.common.PathBasedSecurableObject; import org.apache.gravitino.authorization.ranger.reference.RangerDefines; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.catalog.hive.HiveConstants; import org.apache.gravitino.exceptions.AuthorizationPluginException; +import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.file.Fileset; import org.apache.ranger.plugin.model.RangerPolicy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { - private static final Logger LOG = LoggerFactory.getLogger(RangerAuthorizationHDFSPlugin.class); - private static final Pattern pattern = Pattern.compile("^hdfs://[^/]*"); public RangerAuthorizationHDFSPlugin(String metalake, Map config) { @@ -59,6 +63,38 @@ public RangerAuthorizationHDFSPlugin(String metalake, Map config @Override public Map> privilegesMappingRule() { return ImmutableMap.of( + Privilege.Name.USE_CATALOG, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.CREATE_CATALOG, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.USE_SCHEMA, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.CREATE_SCHEMA, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.CREATE_TABLE, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.MODIFY_TABLE, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.SELECT_TABLE, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), Privilege.Name.READ_FILESET, ImmutableSet.of( RangerPrivileges.RangerHdfsPrivilege.READ, @@ -99,9 +135,9 @@ public AuthorizationSecurableObject generateAuthorizationSecurableObject( AuthorizationMetadataObject.Type type, Set privileges) { AuthorizationMetadataObject authMetadataObject = - new RangerPathBaseMetadataObject(AuthorizationMetadataObject.getLastName(names), type); + new PathBasedMetadataObject(AuthorizationMetadataObject.getLastName(names), type); authMetadataObject.validateAuthorizationMetadataObject(); - return new RangerPathBaseSecurableObject( + return new PathBasedSecurableObject( authMetadataObject.name(), authMetadataObject.type(), privileges); } @@ -137,10 +173,52 @@ public List translatePrivilege(SecurableObject sec .forEach( rangerPrivilege -> rangerPrivileges.add( - new RangerPrivileges.RangerHivePrivilegeImpl( + new RangerPrivileges.RangerHDFSPrivilegeImpl( rangerPrivilege, gravitinoPrivilege.condition()))); - switch (gravitinoPrivilege.name()) { + case USE_CATALOG: + case CREATE_CATALOG: + // When HDFS is used as the Hive storage layer, Hive does not support the + // `USE_CATALOG` and `CREATE_CATALOG` privileges. So, we ignore these + // in the RangerAuthorizationHDFSPlugin. + break; + case USE_SCHEMA: + break; + case CREATE_SCHEMA: + switch (securableObject.type()) { + case METALAKE: + case CATALOG: + { + String locationPath = getLocationPath(securableObject); + if (locationPath != null && !locationPath.isEmpty()) { + PathBasedMetadataObject rangerPathBaseMetadataObject = + new PathBasedMetadataObject( + locationPath, PathBasedMetadataObject.Type.PATH); + rangerSecurableObjects.add( + generateAuthorizationSecurableObject( + rangerPathBaseMetadataObject.names(), + PathBasedMetadataObject.Type.PATH, + rangerPrivileges)); + } + } + break; + case FILESET: + rangerSecurableObjects.add( + generateAuthorizationSecurableObject( + translateMetadataObject(securableObject).names(), + PathBasedMetadataObject.Type.PATH, + rangerPrivileges)); + break; + default: + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); + } + break; + case SELECT_TABLE: + case CREATE_TABLE: + case MODIFY_TABLE: + break; case CREATE_FILESET: // Ignore the Gravitino privilege `CREATE_FILESET` in the // RangerAuthorizationHDFSPlugin @@ -156,7 +234,7 @@ public List translatePrivilege(SecurableObject sec rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(securableObject).names(), - RangerPathBaseMetadataObject.Type.PATH, + PathBasedMetadataObject.Type.PATH, rangerPrivileges)); break; default: @@ -166,10 +244,9 @@ public List translatePrivilege(SecurableObject sec } break; default: - LOG.warn( - "RangerAuthorizationHDFSPlugin -> privilege {} is not supported for the securable object: {}", - gravitinoPrivilege.name(), - securableObject.type()); + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); } }); @@ -183,12 +260,12 @@ public List translateOwner(MetadataObject gravitin case METALAKE: case CATALOG: case SCHEMA: - return rangerSecurableObjects; + break; case FILESET: rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(gravitinoMetadataObject).names(), - RangerPathBaseMetadataObject.Type.PATH, + PathBasedMetadataObject.Type.PATH, ownerMappingRule())); break; default: @@ -212,27 +289,77 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada Preconditions.checkArgument( nsMetadataObject.size() > 0, "The metadata object must have at least one name."); - if (metadataObject.type() == MetadataObject.Type.FILESET) { - RangerPathBaseMetadataObject rangerHDFSMetadataObject = - new RangerPathBaseMetadataObject( - getFileSetPath(metadataObject), RangerPathBaseMetadataObject.Type.PATH); - rangerHDFSMetadataObject.validateAuthorizationMetadataObject(); - return rangerHDFSMetadataObject; - } else { - return new RangerPathBaseMetadataObject("", RangerPathBaseMetadataObject.Type.PATH); + PathBasedMetadataObject rangerPathBaseMetadataObject; + switch (metadataObject.type()) { + case METALAKE: + case CATALOG: + rangerPathBaseMetadataObject = + new PathBasedMetadataObject("", PathBasedMetadataObject.Type.PATH); + break; + case SCHEMA: + rangerPathBaseMetadataObject = + new PathBasedMetadataObject( + metadataObject.fullName(), PathBasedMetadataObject.Type.PATH); + break; + case FILESET: + rangerPathBaseMetadataObject = + new PathBasedMetadataObject( + getLocationPath(metadataObject), PathBasedMetadataObject.Type.PATH); + break; + default: + throw new AuthorizationPluginException( + "The metadata object type %s is not supported in the RangerAuthorizationHDFSPlugin", + metadataObject.type()); } + rangerPathBaseMetadataObject.validateAuthorizationMetadataObject(); + return rangerPathBaseMetadataObject; } - public String getFileSetPath(MetadataObject metadataObject) { - FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); - NameIdentifier identifier = - NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); - Fileset fileset = filesetDispatcher.loadFileset(identifier); - Preconditions.checkArgument( - fileset != null, String.format("Fileset %s is not found", identifier)); - String filesetLocation = fileset.storageLocation(); - Preconditions.checkArgument( - filesetLocation != null, String.format("Fileset %s location is not found", identifier)); - return pattern.matcher(filesetLocation).replaceAll(""); + private NameIdentifier getObjectNameIdentifier(MetadataObject metadataObject) { + return NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); + } + + @VisibleForTesting + public String getLocationPath(MetadataObject metadataObject) throws NoSuchEntityException { + String locationPath = null; + switch (metadataObject.type()) { + case METALAKE: + case SCHEMA: + case TABLE: + break; + case CATALOG: + { + Namespace nsMetadataObj = Namespace.fromString(metadataObject.fullName()); + NameIdentifier ident = NameIdentifier.of(metalake, nsMetadataObj.level(0)); + Catalog catalog = GravitinoEnv.getInstance().catalogDispatcher().loadCatalog(ident); + if (catalog.provider().equals("hive")) { + Schema schema = + GravitinoEnv.getInstance() + .schemaDispatcher() + .loadSchema( + NameIdentifier.of( + metalake, nsMetadataObj.level(0), "default" /*Hive default schema*/)); + String defaultSchemaLocation = schema.properties().get(HiveConstants.LOCATION); + locationPath = pattern.matcher(defaultSchemaLocation).replaceAll(""); + } + } + break; + case FILESET: + FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); + NameIdentifier identifier = getObjectNameIdentifier(metadataObject); + Fileset fileset = filesetDispatcher.loadFileset(identifier); + Preconditions.checkArgument( + fileset != null, String.format("Fileset %s is not found", identifier)); + String filesetLocation = fileset.storageLocation(); + Preconditions.checkArgument( + filesetLocation != null, String.format("Fileset %s location is not found", identifier)); + locationPath = pattern.matcher(filesetLocation).replaceAll(""); + break; + default: + throw new AuthorizationPluginException( + "The metadata object type %s is not supported in the RangerAuthorizationHDFSPlugin", + metadataObject.type()); + } + return locationPath; } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java index b8e078d086e..aab19d31f36 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java @@ -154,25 +154,25 @@ public Set allowMetadataObjectTypesRule() { /** Translate the Gravitino securable object to the Ranger owner securable object. */ @Override public List translateOwner(MetadataObject gravitinoMetadataObject) { - List AuthorizationSecurableObjects = new ArrayList<>(); + List rangerSecurableObjects = new ArrayList<>(); switch (gravitinoMetadataObject.type()) { case METALAKE: case CATALOG: // Add `*` for the SCHEMA permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.SCHEMA, ownerMappingRule())); // Add `*.*` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `*.*.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( RangerHelper.RESOURCE_ALL, @@ -183,20 +183,20 @@ public List translateOwner(MetadataObject gravitin break; case SCHEMA: // Add `{schema}` for the SCHEMA permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(gravitinoMetadataObject.name() /*Schema name*/), RangerHadoopSQLMetadataObject.Type.SCHEMA, ownerMappingRule())); // Add `{schema}.*` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( gravitinoMetadataObject.name() /*Schema name*/, RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `{schema}.*.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( gravitinoMetadataObject.name() /*Schema name*/, @@ -207,13 +207,13 @@ public List translateOwner(MetadataObject gravitin break; case TABLE: // Add `{schema}.{table}` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(gravitinoMetadataObject).names(), RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `{schema}.{table}.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( Stream.concat( translateMetadataObject(gravitinoMetadataObject).names().stream(), @@ -228,13 +228,13 @@ public List translateOwner(MetadataObject gravitin gravitinoMetadataObject.type()); } - return AuthorizationSecurableObjects; + return rangerSecurableObjects; } /** Translate the Gravitino securable object to the Ranger securable object. */ @Override public List translatePrivilege(SecurableObject securableObject) { - List AuthorizationSecurableObjects = new ArrayList<>(); + List rangerSecurableObjects = new ArrayList<>(); securableObject.privileges().stream() .filter(Objects::nonNull) @@ -262,7 +262,7 @@ public List translatePrivilege(SecurableObject sec case METALAKE: case CATALOG: // Add Ranger privilege(`SELECT`) to SCHEMA(`*`) - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.SCHEMA, @@ -279,7 +279,7 @@ public List translatePrivilege(SecurableObject sec case METALAKE: case CATALOG: // Add Ranger privilege(`CREATE`) to SCHEMA(`*`) - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.SCHEMA, @@ -296,7 +296,7 @@ public List translatePrivilege(SecurableObject sec case METALAKE: case CATALOG: // Add Ranger privilege(`SELECT`) to SCHEMA(`*`) - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.SCHEMA, @@ -304,7 +304,7 @@ public List translatePrivilege(SecurableObject sec break; case SCHEMA: // Add Ranger privilege(`SELECT`) to SCHEMA(`{schema}`) - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(securableObject.name() /*Schema name*/), RangerHadoopSQLMetadataObject.Type.SCHEMA, @@ -323,14 +323,14 @@ public List translatePrivilege(SecurableObject sec case METALAKE: case CATALOG: // Add `*.*` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `*.*.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( RangerHelper.RESOURCE_ALL, @@ -341,7 +341,7 @@ public List translatePrivilege(SecurableObject sec break; case SCHEMA: // Add `{schema}.*` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( securableObject.name() /*Schema name*/, @@ -349,7 +349,7 @@ public List translatePrivilege(SecurableObject sec RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `{schema}.*.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( securableObject.name() /*Schema name*/, @@ -365,13 +365,13 @@ public List translatePrivilege(SecurableObject sec gravitinoPrivilege.name(), securableObject.type()); } else { // Add `{schema}.{table}` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(securableObject).names(), RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `{schema}.{table}.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( Stream.concat( translateMetadataObject(securableObject).names().stream(), @@ -396,7 +396,7 @@ public List translatePrivilege(SecurableObject sec } }); - return AuthorizationSecurableObjects; + return rangerSecurableObjects; } /** diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 7a91ad54bf0..1198b68cb46 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -48,6 +48,7 @@ import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.authorization.ranger.reference.VXGroup; import org.apache.gravitino.authorization.ranger.reference.VXGroupList; import org.apache.gravitino.authorization.ranger.reference.VXUser; @@ -87,7 +88,9 @@ public abstract class RangerAuthorizationPlugin protected RangerAuthorizationPlugin(String metalake, Map config) { this.metalake = metalake; - RangerAuthorizationProperties.validate(config); + RangerAuthorizationProperties rangerAuthorizationProperties = + new RangerAuthorizationProperties(config); + rangerAuthorizationProperties.validate(); String rangerUrl = config.get(RangerAuthorizationProperties.RANGER_ADMIN_URL); String authType = config.get(RangerAuthorizationProperties.RANGER_AUTH_TYPE); rangerAdminName = config.get(RangerAuthorizationProperties.RANGER_USERNAME); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java index 8462a0e07a5..d64433b9feb 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java @@ -53,7 +53,7 @@ public static Type fromMetadataType(MetadataObject.Type metadataType) { } } throw new IllegalArgumentException( - "No matching RangerMetadataObject.Type for " + metadataType); + "No matching RangerHadoopSQLMetadataObject.Type for " + metadataType); } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java index 4c2b2956c8c..64c454de61a 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java @@ -49,7 +49,7 @@ public class RangerHelper { private static final Logger LOG = LoggerFactory.getLogger(RangerHelper.class); public static final String MANAGED_BY_GRAVITINO = "MANAGED_BY_GRAVITINO"; - /** The `*` gives access to all resources */ + /** The `*` gives access to all table resources */ public static final String RESOURCE_ALL = "*"; /** The owner privileges, the owner can do anything on the metadata object */ private final Set ownerPrivileges; diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java index bbae16a6ba2..888d98f37d1 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java @@ -116,6 +116,32 @@ public boolean equalsTo(String value) { } } + public static class RangerHDFSPrivilegeImpl implements AuthorizationPrivilege { + private AuthorizationPrivilege rangerHDFSPrivilege; + private Privilege.Condition condition; + + public RangerHDFSPrivilegeImpl( + AuthorizationPrivilege rangerHivePrivilege, Privilege.Condition condition) { + this.rangerHDFSPrivilege = rangerHivePrivilege; + this.condition = condition; + } + + @Override + public String getName() { + return rangerHDFSPrivilege.getName(); + } + + @Override + public Privilege.Condition condition() { + return condition; + } + + @Override + public boolean equalsTo(String value) { + return rangerHDFSPrivilege.equalsTo(value); + } + } + static List>> allRangerPrivileges = Lists.newArrayList( RangerHadoopSQLPrivilege.class, RangerPrivileges.RangerHdfsPrivilege.class); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index 4062263222b..4606fa68e70 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -27,8 +27,8 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.common.PathBasedMetadataObject; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; -import org.apache.gravitino.authorization.ranger.RangerPathBaseMetadataObject; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -56,20 +56,19 @@ public void testTranslateMetadataObject() { MetadataObject metalake = MetadataObjects.parse(String.format("metalake1"), MetadataObject.Type.METALAKE); Assertions.assertEquals( - RangerPathBaseMetadataObject.Type.PATH, + PathBasedMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(metalake).type()); MetadataObject catalog = MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); Assertions.assertEquals( - RangerPathBaseMetadataObject.Type.PATH, + PathBasedMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(catalog).type()); MetadataObject schema = MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); Assertions.assertEquals( - RangerPathBaseMetadataObject.Type.PATH, - rangerAuthPlugin.translateMetadataObject(schema).type()); + PathBasedMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(schema).type()); MetadataObject table = MetadataObjects.parse(String.format("catalog1.schema1.tab1"), MetadataObject.Type.TABLE); @@ -82,7 +81,7 @@ public void testTranslateMetadataObject() { AuthorizationMetadataObject rangerFileset = rangerAuthPlugin.translateMetadataObject(fileset); Assertions.assertEquals(1, rangerFileset.names().size()); Assertions.assertEquals("/test", rangerFileset.fullName()); - Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, rangerFileset.type()); + Assertions.assertEquals(PathBasedMetadataObject.Type.PATH, rangerFileset.type()); } @Test @@ -137,7 +136,7 @@ public void testTranslatePrivilege() { filesetInFileset1.forEach( securableObject -> { - Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, securableObject.type()); + Assertions.assertEquals(PathBasedMetadataObject.Type.PATH, securableObject.type()); Assertions.assertEquals("/test", securableObject.fullName()); Assertions.assertEquals(2, securableObject.privileges().size()); }); @@ -166,7 +165,7 @@ public void testTranslateOwner() { List filesetOwner = rangerAuthPlugin.translateOwner(fileset); Assertions.assertEquals(1, filesetOwner.size()); Assertions.assertEquals("/test", filesetOwner.get(0).fullName()); - Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, filesetOwner.get(0).type()); + Assertions.assertEquals(PathBasedMetadataObject.Type.PATH, filesetOwner.get(0).type()); Assertions.assertEquals(3, filesetOwner.get(0).privileges().size()); } } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java index 1fb9677d528..919551bd922 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java @@ -63,7 +63,6 @@ public abstract class RangerBaseE2EIT extends BaseIT { protected static GravitinoMetalake metalake; protected static Catalog catalog; protected static String HIVE_METASTORE_URIS; - protected static String RANGER_ADMIN_URL = null; protected static SparkSession sparkSession = null; protected static final String HADOOP_USER_NAME = "HADOOP_USER_NAME"; @@ -104,13 +103,13 @@ public abstract class RangerBaseE2EIT extends BaseIT { protected static final String SQL_DROP_TABLE = String.format("DROP TABLE %s", tableName); - protected static void generateRangerSparkSecurityXML() throws IOException { + protected static void generateRangerSparkSecurityXML(String modeName) throws IOException { String templatePath = String.join( File.separator, System.getenv("GRAVITINO_ROOT_DIR"), "authorizations", - "authorization-ranger", + modeName, "src", "test", "resources", @@ -120,7 +119,7 @@ protected static void generateRangerSparkSecurityXML() throws IOException { File.separator, System.getenv("GRAVITINO_ROOT_DIR"), "authorizations", - "authorization-ranger", + modeName, "build", "resources", "test", @@ -130,7 +129,7 @@ protected static void generateRangerSparkSecurityXML() throws IOException { FileUtils.readFileToString(new File(templatePath), StandardCharsets.UTF_8); templateContext = templateContext - .replace("__REPLACE__RANGER_ADMIN_URL", RANGER_ADMIN_URL) + .replace("__REPLACE__RANGER_ADMIN_URL", RangerITEnv.RANGER_ADMIN_URL) .replace("__REPLACE__RANGER_HIVE_REPO_NAME", RangerITEnv.RANGER_HIVE_REPO_NAME); FileUtils.writeStringToFile(new File(xmlPath), templateContext, StandardCharsets.UTF_8); } @@ -220,7 +219,7 @@ void testRenameMetalakeOrCatalog() { } @Test - protected void testCreateSchema() throws InterruptedException { + protected void testCreateSchema() throws InterruptedException, IOException { // Choose a catalog useCatalog(); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java index d8024afcc11..ab74b0449ae 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java @@ -45,7 +45,7 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.authorization.ranger.RangerHelper; import org.apache.gravitino.authorization.ranger.RangerPrivileges; import org.apache.gravitino.client.GravitinoMetalake; diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index 363f8f0b3a1..56cec3e9da3 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -20,7 +20,6 @@ import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; -import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -29,7 +28,7 @@ import org.apache.gravitino.Configs; import org.apache.gravitino.auth.AuthConstants; import org.apache.gravitino.auth.AuthenticatorType; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.catalog.hive.HiveConstants; import org.apache.gravitino.exceptions.UserAlreadyExistsException; import org.apache.gravitino.integration.test.container.HiveContainer; @@ -67,18 +66,13 @@ public void startIntegrationTest() throws Exception { RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); - RANGER_ADMIN_URL = - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); - HIVE_METASTORE_URIS = String.format( "thrift://%s:%d", containerSuite.getHiveRangerContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); - generateRangerSparkSecurityXML(); + generateRangerSparkSecurityXML("authorization-ranger"); sparkSession = SparkSession.builder() @@ -186,7 +180,7 @@ public void createCatalog() { RangerAuthorizationProperties.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HIVE_REPO_NAME, RangerAuthorizationProperties.RANGER_ADMIN_URL, - RANGER_ADMIN_URL, + RangerITEnv.RANGER_ADMIN_URL, RangerAuthorizationProperties.RANGER_AUTH_TYPE, RangerContainer.authType, RangerAuthorizationProperties.RANGER_USERNAME, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index 2efc1e9dd60..913482ef03e 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.authorization.ranger.integration.test; +import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; import static org.mockito.Mockito.doReturn; import com.google.common.collect.ImmutableList; @@ -32,10 +33,10 @@ import org.apache.gravitino.authorization.AuthorizationSecurableObject; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.authorization.ranger.RangerAuthorizationHDFSPlugin; import org.apache.gravitino.authorization.ranger.RangerAuthorizationHadoopSQLPlugin; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; import org.apache.gravitino.authorization.ranger.RangerHelper; import org.apache.gravitino.authorization.ranger.RangerPrivileges; import org.apache.gravitino.authorization.ranger.reference.RangerDefines; @@ -87,11 +88,15 @@ public class RangerITEnv { public static RangerAuthorizationPlugin rangerAuthHivePlugin; public static RangerAuthorizationPlugin rangerAuthHDFSPlugin; protected static RangerHelper rangerHelper; - protected static RangerHelper rangerHDFSHelper; + public static String RANGER_ADMIN_URL = null; public static void init(String metalakeName, boolean allowAnyoneAccessHDFS) { containerSuite.startRangerContainer(); + RANGER_ADMIN_URL = + String.format( + "http://%s:%d", + containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); rangerClient = containerSuite.getRangerContainer().rangerClient; rangerAuthHivePlugin = @@ -134,7 +139,7 @@ public static void init(String metalakeName, boolean allowAnyoneAccessHDFS) { "HDFS", RangerAuthorizationProperties.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HDFS_REPO_NAME))); - doReturn("/test").when(spyRangerAuthorizationHDFSPlugin).getFileSetPath(Mockito.any()); + doReturn("/test").when(spyRangerAuthorizationHDFSPlugin).getLocationPath(Mockito.any()); rangerAuthHDFSPlugin = spyRangerAuthorizationHDFSPlugin; rangerHelper = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java index 8f6f769504a..3e3d0d24234 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java @@ -21,7 +21,6 @@ import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; -import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -35,7 +34,7 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.RangerContainer; @@ -70,18 +69,13 @@ public void startIntegrationTest() throws Exception { RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); - RANGER_ADMIN_URL = - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); - HIVE_METASTORE_URIS = String.format( "thrift://%s:%d", containerSuite.getHiveRangerContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); - generateRangerSparkSecurityXML(); + generateRangerSparkSecurityXML("authorization-ranger"); sparkSession = SparkSession.builder() @@ -179,7 +173,7 @@ public void createCatalog() { properties.put(RangerAuthorizationProperties.RANGER_SERVICE_TYPE, "HadoopSQL"); properties.put( RangerAuthorizationProperties.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HIVE_REPO_NAME); - properties.put(RangerAuthorizationProperties.RANGER_ADMIN_URL, RANGER_ADMIN_URL); + properties.put(RangerAuthorizationProperties.RANGER_ADMIN_URL, RangerITEnv.RANGER_ADMIN_URL); properties.put(RangerAuthorizationProperties.RANGER_AUTH_TYPE, RangerContainer.authType); properties.put(RangerAuthorizationProperties.RANGER_USERNAME, RangerContainer.rangerUserName); properties.put(RangerAuthorizationProperties.RANGER_PASSWORD, RangerContainer.rangerPassword); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java index 2773610048e..c37fd20c85f 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java @@ -20,7 +20,6 @@ import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; -import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -34,7 +33,7 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.RangerContainer; import org.apache.gravitino.integration.test.util.GravitinoITUtils; @@ -69,18 +68,13 @@ public void startIntegrationTest() throws Exception { RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); - RANGER_ADMIN_URL = - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); - HIVE_METASTORE_URIS = String.format( "thrift://%s:%d", containerSuite.getHiveRangerContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); - generateRangerSparkSecurityXML(); + generateRangerSparkSecurityXML("authorization-ranger"); sparkSession = SparkSession.builder() @@ -199,7 +193,7 @@ public void createCatalog() { RangerAuthorizationProperties.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HIVE_REPO_NAME, RangerAuthorizationProperties.RANGER_ADMIN_URL, - RANGER_ADMIN_URL, + RangerITEnv.RANGER_ADMIN_URL, RangerAuthorizationProperties.RANGER_AUTH_TYPE, RangerContainer.authType, RangerAuthorizationProperties.RANGER_USERNAME, diff --git a/authorizations/build.gradle.kts b/authorizations/build.gradle.kts index 043fbfec673..354b36aae64 100644 --- a/authorizations/build.gradle.kts +++ b/authorizations/build.gradle.kts @@ -17,6 +17,18 @@ * under the License. */ -tasks.all { - enabled = false -} \ No newline at end of file +tasks { + test { + subprojects.forEach { + dependsOn(":${project.name}:${it.name}:test") + } + } + + register("copyLibAndConfig", Copy::class) { + subprojects.forEach { + if (!it.name.startsWith("authorization-common")) { + dependsOn(":${project.name}:${it.name}:copyLibAndConfig") + } + } + } +} diff --git a/build.gradle.kts b/build.gradle.kts index 5e93992e34e..c64997f3a90 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -583,7 +583,7 @@ tasks { val outputDir = projectDir.dir("distribution") val compileDistribution by registering { - dependsOn(":web:web:build", "copySubprojectDependencies", "copyCatalogLibAndConfigs", "copyAuthorizationLibAndConfigs", "copySubprojectLib", "iceberg:iceberg-rest-server:copyLibAndConfigs") + dependsOn(":web:web:build", "copySubprojectDependencies", "copyCatalogLibAndConfigs", ":authorizations:copyLibAndConfig", "copySubprojectLib", "iceberg:iceberg-rest-server:copyLibAndConfigs") group = "gravitino distribution" outputs.dir(projectDir.dir("distribution/package")) @@ -829,12 +829,6 @@ tasks { ) } - register("copyAuthorizationLibAndConfigs", Copy::class) { - dependsOn( - ":authorizations:authorization-ranger:copyLibAndConfig" - ) - } - clean { dependsOn(cleanDistribution) } diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java index 3d71948b744..36307f3ba4b 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import java.lang.reflect.Field; import java.time.LocalDate; import java.util.Collections; import java.util.Map; @@ -423,21 +422,4 @@ private static void loadCatalogWithAnotherClient() { anotherCatalogWithNotExistingName = anotherClientWithNotExistingName.loadMetalake(METALAKE_NAME).loadCatalog(CATALOG_NAME); } - - public static void setEnv(String key, String value) { - try { - Map env = System.getenv(); - Class cl = env.getClass(); - Field field = cl.getDeclaredField("m"); - field.setAccessible(true); - Map writableEnv = (Map) field.get(env); - if (value == null) { - writableEnv.remove(key); - } else { - writableEnv.put(key, value); - } - } catch (Exception e) { - throw new IllegalStateException("Failed to set environment variable", e); - } - } } diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index 43bc74bb2a1..4a46952f87e 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -58,7 +58,6 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.Catalog; import org.apache.gravitino.CatalogChange; @@ -79,6 +78,7 @@ import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.PropertyEntry; import org.apache.gravitino.connector.SupportsSchemas; +import org.apache.gravitino.connector.authorization.BaseAuthorization; import org.apache.gravitino.connector.capability.Capability; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; import org.apache.gravitino.exceptions.CatalogInUseException; @@ -944,7 +944,7 @@ private IsolatedClassLoader createClassLoader(String provider, Map libAndResourcesPaths = Lists.newArrayList(catalogPkgPath, catalogConfPath); - buildAuthorizationPkgPath(conf).ifPresent(libAndResourcesPaths::add); + BaseAuthorization.buildAuthorizationPkgPath(conf).ifPresent(libAndResourcesPaths::add); return IsolatedClassLoader.buildClassLoader(libAndResourcesPaths); } else { // This will use the current class loader, it is mainly used for test. @@ -1061,37 +1061,6 @@ private String buildPkgPath(Map conf, String provider) { return pkgPath; } - private Optional buildAuthorizationPkgPath(Map conf) { - String gravitinoHome = System.getenv("GRAVITINO_HOME"); - Preconditions.checkArgument(gravitinoHome != null, "GRAVITINO_HOME not set"); - boolean testEnv = System.getenv("GRAVITINO_TEST") != null; - - String authorizationProvider = conf.get(Catalog.AUTHORIZATION_PROVIDER); - if (StringUtils.isBlank(authorizationProvider)) { - return Optional.empty(); - } - - String pkgPath; - if (testEnv) { - // In test, the authorization package is under the build directory. - pkgPath = - String.join( - File.separator, - gravitinoHome, - "authorizations", - "authorization-" + authorizationProvider, - "build", - "libs"); - } else { - // In real environment, the authorization package is under the authorization directory. - pkgPath = - String.join( - File.separator, gravitinoHome, "authorizations", authorizationProvider, "libs"); - } - - return Optional.of(pkgPath); - } - private Class lookupCatalogProvider(String provider, ClassLoader cl) { ServiceLoader loader = ServiceLoader.load(CatalogProvider.class, cl); diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java index 88fd47ab998..218c2a428b3 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java @@ -19,22 +19,16 @@ package org.apache.gravitino.connector; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import com.google.common.collect.Streams; import java.io.Closeable; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.ServiceLoader; -import java.util.stream.Collectors; import org.apache.gravitino.Audit; import org.apache.gravitino.Catalog; import org.apache.gravitino.CatalogProvider; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; -import org.apache.gravitino.connector.authorization.AuthorizationProvider; import org.apache.gravitino.connector.authorization.BaseAuthorization; import org.apache.gravitino.connector.capability.Capability; import org.apache.gravitino.meta.CatalogEntity; @@ -209,34 +203,7 @@ public void initAuthorizationPluginInstance(IsolatedClassLoader classLoader) { } try { BaseAuthorization authorization = - classLoader.withClassLoader( - cl -> { - try { - ServiceLoader loader = - ServiceLoader.load(AuthorizationProvider.class, cl); - - List> providers = - Streams.stream(loader.iterator()) - .filter(p -> p.shortName().equalsIgnoreCase(authorizationProvider)) - .map(AuthorizationProvider::getClass) - .collect(Collectors.toList()); - if (providers.isEmpty()) { - throw new IllegalArgumentException( - "No authorization provider found for: " + authorizationProvider); - } else if (providers.size() > 1) { - throw new IllegalArgumentException( - "Multiple authorization providers found for: " - + authorizationProvider); - } - return (BaseAuthorization) - Iterables.getOnlyElement(providers) - .getDeclaredConstructor() - .newInstance(); - } catch (Exception e) { - LOG.error("Failed to create authorization instance", e); - throw new RuntimeException(e); - } - }); + BaseAuthorization.createAuthorization(classLoader, authorizationProvider); authorizationPlugin = authorization.newPlugin(entity.namespace().level(0), provider(), this.conf); } catch (Exception e) { diff --git a/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java b/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java index 173ad3527a8..740f808e4df 100644 --- a/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java +++ b/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java @@ -18,9 +18,20 @@ */ package org.apache.gravitino.connector.authorization; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.common.collect.Streams; import java.io.Closeable; +import java.io.File; import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.utils.IsolatedClassLoader; /** * The abstract base class for Authorization implementations.
@@ -46,4 +57,65 @@ public abstract AuthorizationPlugin newPlugin( @Override public void close() throws IOException {} + + public static BaseAuthorization createAuthorization( + IsolatedClassLoader classLoader, String authorizationProvider) throws Exception { + BaseAuthorization authorization = + classLoader.withClassLoader( + cl -> { + try { + ServiceLoader loader = + ServiceLoader.load(AuthorizationProvider.class, cl); + + List> providers = + Streams.stream(loader.iterator()) + .filter(p -> p.shortName().equalsIgnoreCase(authorizationProvider)) + .map(AuthorizationProvider::getClass) + .collect(Collectors.toList()); + if (providers.isEmpty()) { + throw new IllegalArgumentException( + "No authorization provider found for: " + authorizationProvider); + } else if (providers.size() > 1) { + throw new IllegalArgumentException( + "Multiple authorization providers found for: " + authorizationProvider); + } + return (BaseAuthorization) + Iterables.getOnlyElement(providers).getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return authorization; + } + + public static Optional buildAuthorizationPkgPath(Map conf) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + Preconditions.checkArgument(gravitinoHome != null, "GRAVITINO_HOME not set"); + boolean testEnv = System.getenv("GRAVITINO_TEST") != null; + + String authorizationProvider = conf.get(Catalog.AUTHORIZATION_PROVIDER); + if (StringUtils.isBlank(authorizationProvider)) { + return Optional.empty(); + } + + String pkgPath; + if (testEnv) { + // In test, the authorization package is under the build directory. + pkgPath = + String.join( + File.separator, + gravitinoHome, + "authorizations", + "authorization-" + authorizationProvider, + "build", + "libs"); + } else { + // In real environment, the authorization package is under the authorization directory. + pkgPath = + String.join( + File.separator, gravitinoHome, "authorizations", authorizationProvider, "libs"); + } + + return Optional.of(pkgPath); + } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/TestAuthorization.java b/core/src/test/java/org/apache/gravitino/connector/authorization/TestAuthorization.java index 554ef0cec8b..4ee37b4ddec 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/TestAuthorization.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/TestAuthorization.java @@ -24,8 +24,8 @@ import org.apache.gravitino.Catalog; import org.apache.gravitino.Namespace; import org.apache.gravitino.TestCatalog; -import org.apache.gravitino.connector.authorization.mysql.TestMySQLAuthorizationPlugin; -import org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorizationPlugin; +import org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorizationHDFSPlugin; +import org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorizationHadoopSQLPlugin; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.utils.IsolatedClassLoader; @@ -35,7 +35,7 @@ public class TestAuthorization { private static TestCatalog hiveCatalog; - private static TestCatalog mySQLCatalog; + private static TestCatalog filesetCatalog; @BeforeAll public static void setUp() throws Exception { @@ -54,49 +54,59 @@ public static void setUp() throws Exception { hiveCatalog = new TestCatalog() - .withCatalogConf(ImmutableMap.of(Catalog.AUTHORIZATION_PROVIDER, "ranger")) + .withCatalogConf( + ImmutableMap.of( + Catalog.AUTHORIZATION_PROVIDER, + "test-ranger", + "authorization.ranger.service.type", + "HadoopSQL")) .withCatalogEntity(hiveCatalogEntity); IsolatedClassLoader isolatedClassLoader = new IsolatedClassLoader( Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); hiveCatalog.initAuthorizationPluginInstance(isolatedClassLoader); - CatalogEntity mySQLEntity = + CatalogEntity filesetEntity = CatalogEntity.builder() .withId(2L) .withName("catalog-test2") .withNamespace(Namespace.of("default")) - .withType(Catalog.Type.RELATIONAL) + .withType(Catalog.Type.FILESET) .withProvider("test") .withAuditInfo(auditInfo) .build(); - mySQLCatalog = + filesetCatalog = new TestCatalog() - .withCatalogConf(ImmutableMap.of(Catalog.AUTHORIZATION_PROVIDER, "mysql")) - .withCatalogEntity(mySQLEntity); - mySQLCatalog.initAuthorizationPluginInstance(isolatedClassLoader); + .withCatalogConf( + ImmutableMap.of( + Catalog.AUTHORIZATION_PROVIDER, + "test-ranger", + "authorization.ranger.service.type", + "HDFS")) + .withCatalogEntity(filesetEntity); + filesetCatalog.initAuthorizationPluginInstance(isolatedClassLoader); } @Test - public void testRangerAuthorization() { - AuthorizationPlugin rangerAuthPlugin = hiveCatalog.getAuthorizationPlugin(); - Assertions.assertInstanceOf(TestRangerAuthorizationPlugin.class, rangerAuthPlugin); - TestRangerAuthorizationPlugin testRangerAuthPlugin = - (TestRangerAuthorizationPlugin) rangerAuthPlugin; - Assertions.assertFalse(testRangerAuthPlugin.callOnCreateRole1); - rangerAuthPlugin.onRoleCreated(null); - Assertions.assertTrue(testRangerAuthPlugin.callOnCreateRole1); + public void testRangerHadoopSQLAuthorization() { + AuthorizationPlugin rangerHiveAuthPlugin = hiveCatalog.getAuthorizationPlugin(); + Assertions.assertInstanceOf(TestRangerAuthorizationHadoopSQLPlugin.class, rangerHiveAuthPlugin); + TestRangerAuthorizationHadoopSQLPlugin testRangerAuthHadoopSQLPlugin = + (TestRangerAuthorizationHadoopSQLPlugin) rangerHiveAuthPlugin; + Assertions.assertFalse(testRangerAuthHadoopSQLPlugin.callOnCreateRole1); + rangerHiveAuthPlugin.onRoleCreated(null); + Assertions.assertTrue(testRangerAuthHadoopSQLPlugin.callOnCreateRole1); } @Test - public void testMySQLAuthorization() { - AuthorizationPlugin mySQLAuthPlugin = mySQLCatalog.getAuthorizationPlugin(); - Assertions.assertInstanceOf(TestMySQLAuthorizationPlugin.class, mySQLAuthPlugin); - TestMySQLAuthorizationPlugin testMySQLAuthPlugin = - (TestMySQLAuthorizationPlugin) mySQLAuthPlugin; - Assertions.assertFalse(testMySQLAuthPlugin.callOnCreateRole2); - mySQLAuthPlugin.onRoleCreated(null); - Assertions.assertTrue(testMySQLAuthPlugin.callOnCreateRole2); + public void testRangerHDFSAuthorization() { + AuthorizationPlugin rangerHDFSAuthPlugin = filesetCatalog.getAuthorizationPlugin(); + Assertions.assertInstanceOf(TestRangerAuthorizationHDFSPlugin.class, rangerHDFSAuthPlugin); + TestRangerAuthorizationHDFSPlugin testRangerAuthHDFSPlugin = + (TestRangerAuthorizationHDFSPlugin) rangerHDFSAuthPlugin; + Assertions.assertFalse(testRangerAuthHDFSPlugin.callOnCreateRole2); + rangerHDFSAuthPlugin.onRoleCreated(null); + Assertions.assertTrue(testRangerAuthHDFSPlugin.callOnCreateRole2); } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java index 9df9a8d63b7..1709c90319f 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.connector.authorization.ranger; +import com.google.common.base.Preconditions; import java.util.Map; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; @@ -28,12 +29,23 @@ public TestRangerAuthorization() {} @Override public String shortName() { - return "ranger"; + return "test-ranger"; } @Override public AuthorizationPlugin newPlugin( - String metalake, String catalogProvider, Map config) { - return new TestRangerAuthorizationPlugin(); + String metalake, String catalogProvider, Map properties) { + Preconditions.checkArgument( + properties.containsKey("authorization.ranger.service.type"), + String.format("%s is required", "authorization.ranger.service.type")); + String serviceType = properties.get("authorization.ranger.service.type").toUpperCase(); + switch (serviceType) { + case "HADOOPSQL": + return new TestRangerAuthorizationHadoopSQLPlugin(); + case "HDFS": + return new TestRangerAuthorizationHDFSPlugin(); + default: + throw new IllegalArgumentException("Unsupported service type: " + serviceType); + } } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHDFSPlugin.java similarity index 95% rename from core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java rename to core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHDFSPlugin.java index e078eda410e..fdc28f8143e 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHDFSPlugin.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.connector.authorization.mysql; +package org.apache.gravitino.connector.authorization.ranger; import java.io.IOException; import java.util.List; @@ -29,7 +29,7 @@ import org.apache.gravitino.authorization.User; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; -public class TestMySQLAuthorizationPlugin implements AuthorizationPlugin { +public class TestRangerAuthorizationHDFSPlugin implements AuthorizationPlugin { public boolean callOnCreateRole2 = false; @Override diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHadoopSQLPlugin.java similarity index 97% rename from core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java rename to core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHadoopSQLPlugin.java index 8a68f825d0e..10dbe521e6c 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHadoopSQLPlugin.java @@ -29,7 +29,7 @@ import org.apache.gravitino.authorization.User; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; -public class TestRangerAuthorizationPlugin implements AuthorizationPlugin { +public class TestRangerAuthorizationHadoopSQLPlugin implements AuthorizationPlugin { public boolean callOnCreateRole1 = false; @Override diff --git a/core/src/test/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider b/core/src/test/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider index e49cb8937e0..b7219fdc279 100644 --- a/core/src/test/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider +++ b/core/src/test/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider @@ -16,5 +16,4 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorization -org.apache.gravitino.connector.authorization.mysql.TestMySQLAuthorization \ No newline at end of file +org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorization \ No newline at end of file diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index a25ad4cff8f..283169a76a9 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -54,7 +54,10 @@ dependencies { exclude("org.elasticsearch.client") exclude("org.elasticsearch.plugin") } - + testImplementation(libs.hadoop3.common) { + exclude("com.sun.jersey") + exclude("javax.servlet", "servlet-api") + } testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java index fcf8ebb2b9c..f8cbe508f87 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java @@ -26,6 +26,7 @@ import com.google.common.base.Splitter; import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -57,6 +58,7 @@ import org.apache.gravitino.server.GravitinoServer; import org.apache.gravitino.server.ServerConfig; import org.apache.gravitino.server.web.JettyServerConfig; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; @@ -422,4 +424,40 @@ protected static void copyBundleJarsToHadoop(String bundleName) { String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs"); copyBundleJarsToDirectory(bundleName, hadoopLibDirs); } + + public static void runInEnv(String key, String value, Runnable lambda) { + String originalValue = System.getenv(key); + try { + setEnv(key, value); + if (key.equals("HADOOP_USER_NAME") && value != null) { + UserGroupInformation.setLoginUser(null); + System.setProperty("user.name", value); + } + lambda.run(); + } catch (Exception e) { + throw new IllegalStateException("Failed to set environment variable", e); + } finally { + setEnv(key, originalValue); + if (key.equals("HADOOP_USER_NAME") && value != null) { + System.setProperty("user.name", originalValue); + } + } + } + + public static void setEnv(String key, String value) { + try { + Map env = System.getenv(); + Class cl = env.getClass(); + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Map writableEnv = (Map) field.get(env); + if (value == null) { + writableEnv.remove(key); + } else { + writableEnv.put(key, value); + } + } catch (Exception e) { + throw new IllegalStateException("Failed to set environment variable", e); + } + } } diff --git a/settings.gradle.kts b/settings.gradle.kts index b3eb56578aa..f38443db206 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,7 +57,7 @@ if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() == true) } include("iceberg:iceberg-common") include("iceberg:iceberg-rest-server") -include("authorizations:authorization-ranger", "authorizations:authorization-jdbc") +include("authorizations:authorization-ranger", "authorizations:authorization-jdbc", "authorizations:authorization-common", "authorizations:authorization-chain") include("trino-connector:trino-connector", "trino-connector:integration-test") include("spark-connector:spark-common") // kyuubi hive connector doesn't support 2.13 for Spark3.3 From fc7a6b3a98723b7a49453acc5c43f46aa7874464 Mon Sep 17 00:00:00 2001 From: cai can <94670132+caican00@users.noreply.github.com> Date: Wed, 25 Dec 2024 15:49:32 +0800 Subject: [PATCH 32/47] [#4714] feat(paimon-spark-connector): Add tests for partitionManagement of paimon table in paimon spark connector (#5860) ### What changes were proposed in this pull request? Add tests for partitionManagement of paimon table in paimon spark connector ### Why are the changes needed? Fix: https://github.com/apache/gravitino/issues/4714 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? New ITs. --------- Co-authored-by: caican --- .../test/paimon/SparkPaimonCatalogIT.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java index c77a4642eec..9d036482857 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java @@ -104,6 +104,37 @@ void testPaimonPartitions() { checkDirExists(partitionPath); } + @Test + void testPaimonPartitionManagement() { + testPaimonListAndDropPartition(); + // TODO: replace, add and load partition operations are unsupported now. + } + + private void testPaimonListAndDropPartition() { + String tableName = "test_paimon_drop_partition"; + dropTableIfExists(tableName); + String createTableSQL = getCreatePaimonSimpleTableString(tableName); + createTableSQL = createTableSQL + " PARTITIONED BY (name);"; + sql(createTableSQL); + + String insertData = + String.format( + "INSERT into %s values(1,'a','beijing'), (2,'b','beijing'), (3,'c','beijing');", + tableName); + sql(insertData); + List queryResult = getTableData(tableName); + Assertions.assertEquals(3, queryResult.size()); + + List partitions = getQueryData(String.format("show partitions %s", tableName)); + Assertions.assertEquals(3, partitions.size()); + Assertions.assertEquals("name=a;name=b;name=c", String.join(";", partitions)); + + sql(String.format("ALTER TABLE %s DROP PARTITION (`name`='a')", tableName)); + partitions = getQueryData(String.format("show partitions %s", tableName)); + Assertions.assertEquals(2, partitions.size()); + Assertions.assertEquals("name=b;name=c", String.join(";", partitions)); + } + private String getCreatePaimonSimpleTableString(String tableName) { return String.format( "CREATE TABLE %s (id INT COMMENT 'id comment', name STRING COMMENT '', address STRING COMMENT '') USING paimon", From 525f8aae09ae4dbcba0f4db783b37fd4cf76e407 Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Thu, 26 Dec 2024 06:11:31 +0800 Subject: [PATCH 33/47] [#5861] improvement(CLI): Refactor the validation logic in the handle methods (#5972) ### What changes were proposed in this pull request? refactor the validation logic of all entities and add test case, just like validation of table command #5906 . A hint is provided when the user's output is missing the required arguments. for example: ```bash gcli column list -m demo_metalake, --name Hive_catalog # Malformed entity name. # Missing required argument(s): schema, table gcli column details -m demo_metalake, --name Hive_catalog --audit # Malformed entity name. # Missing required argument(s): schema, table, column gcli user delete -m demo_metalake Missing --user option. ``` Currently, the Role command needs to be refactored and opened as a separate issue ### Why are the changes needed? Fix: #5861 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? local test --- .../apache/gravitino/cli/ErrorMessages.java | 1 + .../gravitino/cli/GravitinoCommandLine.java | 87 +++++--- .../gravitino/cli/TestCatalogCommands.java | 33 +++ .../gravitino/cli/TestColumnCommands.java | 211 ++++++++++++++++++ .../gravitino/cli/TestFilesetCommands.java | 155 +++++++++++++ .../gravitino/cli/TestGroupCommands.java | 43 ++++ .../org/apache/gravitino/cli/TestMain.java | 1 - .../gravitino/cli/TestTableCommands.java | 76 ++++--- .../apache/gravitino/cli/TestTagCommands.java | 42 ++++ .../gravitino/cli/TestTopicCommands.java | 154 +++++++++++++ .../gravitino/cli/TestUserCommands.java | 42 ++++ 11 files changed, 778 insertions(+), 67 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java index 3423cee07f7..1d6db1a5acd 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java @@ -32,6 +32,7 @@ public class ErrorMessages { public static final String MISSING_NAME = "Missing --name option."; public static final String MISSING_GROUP = "Missing --group option."; public static final String MISSING_USER = "Missing --user option."; + public static final String MISSING_TAG = "Missing --tag option."; public static final String METALAKE_EXISTS = "Metalake already exists."; public static final String CATALOG_EXISTS = "Catalog already exists."; public static final String SCHEMA_EXISTS = "Schema already exists."; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 3f37e84ff65..bd9cc1f8551 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -31,8 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; @@ -255,6 +253,7 @@ private void handleCatalogCommand() { String outputFormat = line.getOptionValue(GravitinoOptions.OUTPUT); Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); // Handle the CommandActions.LIST action separately as it doesn't use `catalog` if (CommandActions.LIST.equals(command)) { @@ -263,6 +262,8 @@ private void handleCatalogCommand() { } String catalog = name.getCatalogName(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -343,29 +344,21 @@ private void handleSchemaCommand() { String catalog = name.getCatalogName(); Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); if (metalake == null) missingEntities.add(CommandEntities.METALAKE); if (catalog == null) missingEntities.add(CommandEntities.CATALOG); // Handle the CommandActions.LIST action separately as it doesn't use `schema` if (CommandActions.LIST.equals(command)) { - if (!missingEntities.isEmpty()) { - System.err.println("Missing required argument(s): " + COMMA_JOINER.join(missingEntities)); - Main.exit(-1); - } + checkEntities(missingEntities); newListSchema(url, ignore, metalake, catalog).handle(); return; } String schema = name.getSchemaName(); - if (schema == null) { - missingEntities.add(CommandEntities.SCHEMA); - } - - if (!missingEntities.isEmpty()) { - System.err.println("Missing required argument(s): " + COMMA_JOINER.join(missingEntities)); - Main.exit(-1); - } + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -421,33 +414,20 @@ private void handleTableCommand() { String schema = name.getSchemaName(); Command.setAuthenticationMode(auth, userName); - List missingEntities = - Stream.of( - catalog == null ? CommandEntities.CATALOG : null, - schema == null ? CommandEntities.SCHEMA : null) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); // Handle CommandActions.LIST action separately as it doesn't require the `table` if (CommandActions.LIST.equals(command)) { - if (!missingEntities.isEmpty()) { - System.err.println( - "Missing required argument(s): " + Joiner.on(", ").join(missingEntities)); - Main.exit(-1); - } + checkEntities(missingEntities); newListTables(url, ignore, metalake, catalog, schema).handle(); return; } String table = name.getTableName(); - if (table == null) { - missingEntities.add(CommandEntities.TABLE); - } - - if (!missingEntities.isEmpty()) { - System.err.println("Missing required argument(s): " + Joiner.on(", ").join(missingEntities)); - Main.exit(-1); - } + if (table == null) missingEntities.add(CommandEntities.TABLE); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -527,7 +507,7 @@ protected void handleUserCommand() { if (user == null && !CommandActions.LIST.equals(command)) { System.err.println(ErrorMessages.MISSING_USER); - return; + Main.exit(-1); } switch (command) { @@ -588,7 +568,7 @@ protected void handleGroupCommand() { if (group == null && !CommandActions.LIST.equals(command)) { System.err.println(ErrorMessages.MISSING_GROUP); - return; + Main.exit(-1); } switch (command) { @@ -647,6 +627,13 @@ protected void handleTagCommand() { Command.setAuthenticationMode(auth, userName); String[] tags = line.getOptionValues(GravitinoOptions.TAG); + if (tags == null + && !((CommandActions.REMOVE.equals(command) && line.hasOption(GravitinoOptions.FORCE)) + || CommandActions.LIST.equals(command))) { + System.err.println(ErrorMessages.MISSING_TAG); + Main.exit(-1); + } + if (tags != null) { tags = Arrays.stream(tags).distinct().toArray(String[]::new); } @@ -812,12 +799,20 @@ private void handleColumnCommand() { Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + if (table == null) missingEntities.add(CommandEntities.TABLE); + if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); newListColumns(url, ignore, metalake, catalog, schema, table).handle(); return; } String column = name.getColumnName(); + if (column == null) missingEntities.add(CommandEntities.COLUMN); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -987,12 +982,19 @@ private void handleTopicCommand() { Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); newListTopics(url, ignore, metalake, catalog, schema).handle(); return; } String topic = name.getTopicName(); + if (topic == null) missingEntities.add(CommandEntities.TOPIC); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -1062,12 +1064,20 @@ private void handleFilesetCommand() { Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + + // Handle CommandActions.LIST action separately as it doesn't require the `fileset` if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); newListFilesets(url, ignore, metalake, catalog, schema).handle(); return; } String fileset = name.getFilesetName(); + if (fileset == null) missingEntities.add(CommandEntities.FILESET); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -1205,4 +1215,11 @@ public String getAuth() { return null; } + + private void checkEntities(List entities) { + if (!entities.isEmpty()) { + System.err.println("Missing required argument(s): " + COMMA_JOINER.join(entities)); + Main.exit(-1); + } + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java index d751d671731..44e5537955f 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java @@ -19,6 +19,7 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; @@ -30,6 +31,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; @@ -37,6 +39,7 @@ import org.apache.gravitino.cli.commands.CatalogDetails; import org.apache.gravitino.cli.commands.CatalogDisable; import org.apache.gravitino.cli.commands.CatalogEnable; +import org.apache.gravitino.cli.commands.Command; import org.apache.gravitino.cli.commands.CreateCatalog; import org.apache.gravitino.cli.commands.DeleteCatalog; import org.apache.gravitino.cli.commands.ListCatalogProperties; @@ -318,6 +321,36 @@ void testUpdateCatalogNameCommand() { verify(mockUpdateName).handle(); } + @Test + @SuppressWarnings("DefaultCharset") + void testCatalogDetailsCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newCatalogDetails( + GravitinoCommandLine.DEFAULT_URL, + false, + Command.OUTPUT_FORMAT_TABLE, + "metalake_demo", + "catalog"); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + "Missing --name option." + + "\n" + + "Missing required argument(s): " + + CommandEntities.CATALOG); + } + @Test void testEnableCatalogCommand() { CatalogEnable mockEnable = mock(CatalogEnable.class); diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java index 2eb4c536480..b6159343ef0 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java @@ -28,9 +28,11 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.base.Joiner; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.AddColumn; @@ -64,6 +66,11 @@ void setUp() { System.setErr(new PrintStream(errContent)); } + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + @AfterEach public void restoreStreams() { System.setOut(originalOut); @@ -435,4 +442,208 @@ void testUpdateColumnDefault() { commandLine.handleCommandLine(); verify(mockUpdateDefault).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteColumnCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteColumn( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.CATALOG, + CommandEntities.SCHEMA, + CommandEntities.TABLE, + CommandEntities.COLUMN))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteColumnCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteColumn( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.SCHEMA, CommandEntities.TABLE, CommandEntities.COLUMN))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteColumnCommandWithoutTable() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteColumn( + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + "catalog", + "schema", + null, + null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.TABLE, CommandEntities.COLUMN))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteColumnCommandWithoutColumn() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema.users"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteColumn( + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + "catalog", + "schema", + "users", + null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.COLUMN))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListColumnCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListColumns( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.CATALOG, CommandEntities.SCHEMA, CommandEntities.TABLE))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListColumnCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListColumns( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA, CommandEntities.TABLE))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListColumnCommandWithoutTable() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListColumns( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.TABLE); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java index 314e118c7d7..b46b73cc3dd 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java @@ -19,14 +19,22 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.base.Joiner; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateFileset; @@ -38,17 +46,35 @@ import org.apache.gravitino.cli.commands.SetFilesetProperty; import org.apache.gravitino.cli.commands.UpdateFilesetComment; import org.apache.gravitino.cli.commands.UpdateFilesetName; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestFilesetCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -322,4 +348,133 @@ void testRemoveFilesetPropertyCommand() { commandLine.handleCommandLine(); verify(mockSetProperties).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testListFilesetCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListFilesets(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.CATALOG, CommandEntities.SCHEMA))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListFilesetCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListFilesets(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testFilesetDetailCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newFilesetDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.CATALOG, CommandEntities.SCHEMA, CommandEntities.FILESET))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testFilesetDetailCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newFilesetDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA, CommandEntities.FILESET))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testFilesetDetailCommandWithoutFileset() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newFilesetDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.FILESET))); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java index 3f1c4a4cb1e..98e3ea910fb 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java @@ -19,12 +19,18 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.AddRoleToGroup; @@ -34,17 +40,35 @@ import org.apache.gravitino.cli.commands.GroupDetails; import org.apache.gravitino.cli.commands.ListGroups; import org.apache.gravitino.cli.commands.RemoveRoleFromGroup; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestGroupCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -260,4 +284,23 @@ void testAddRolesToGroupCommand() { verify(mockAddSecondRole).handle(); verify(mockAddFirstRole).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteGroupCommandWithoutGroupOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.GROUP)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.GROUP, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteGroup(GravitinoCommandLine.DEFAULT_URL, false, false, "metalake_demo", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_GROUP); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java index 377e569aa53..1d1ffded0ff 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java @@ -192,7 +192,6 @@ public void CreateTagWithNoTag() { assertTrue(errContent.toString().contains(ErrorMessages.TAG_EMPTY)); // Expect error } - @Test @SuppressWarnings("DefaultCharset") public void DeleteTagWithNoTag() { String[] args = {"tag", "delete", "--metalake", "metalake_test_no_tag", "-f"}; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java index 32c289cfd85..c4a8223dd48 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -30,6 +30,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateTable; @@ -451,14 +452,15 @@ void testListTableWithoutCatalog() { assertThrows(RuntimeException.class, commandLine::handleCommandLine); verify(commandLine, never()) .newListTables(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null); - assertTrue( - errContent - .toString() - .contains( - "Missing required argument(s): " - + CommandEntities.CATALOG - + ", " - + CommandEntities.SCHEMA)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.CATALOG + + ", " + + CommandEntities.SCHEMA); } @Test @@ -478,8 +480,13 @@ void testListTableWithoutSchema() { assertThrows(RuntimeException.class, commandLine::handleCommandLine); verify(commandLine, never()) .newListTables(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null); - assertTrue( - errContent.toString().contains("Missing required argument(s): " + CommandEntities.SCHEMA)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.SCHEMA); } @Test @@ -498,16 +505,17 @@ void testDetailTableWithoutCatalog() { verify(commandLine, never()) .newTableDetails( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null); - assertTrue( - errContent - .toString() - .contains( - "Missing required argument(s): " - + CommandEntities.CATALOG - + ", " - + CommandEntities.SCHEMA - + ", " - + CommandEntities.TABLE)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.CATALOG + + ", " + + CommandEntities.SCHEMA + + ", " + + CommandEntities.TABLE); } @Test @@ -526,14 +534,15 @@ void testDetailTableWithoutSchema() { verify(commandLine, never()) .newTableDetails( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null); - assertTrue( - errContent - .toString() - .contains( - "Missing required argument(s): " - + CommandEntities.SCHEMA - + ", " - + CommandEntities.TABLE)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.SCHEMA + + ", " + + CommandEntities.TABLE); } @Test @@ -554,7 +563,12 @@ void testDetailTableWithoutTable() { verify(commandLine, never()) .newTableDetails( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); - assertTrue( - errContent.toString().contains("Missing required argument(s): " + CommandEntities.TABLE)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.TABLE); } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java index b68d6c81050..03c583e24f6 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java @@ -19,15 +19,21 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateTag; @@ -43,6 +49,7 @@ import org.apache.gravitino.cli.commands.UntagEntity; import org.apache.gravitino.cli.commands.UpdateTagComment; import org.apache.gravitino.cli.commands.UpdateTagName; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -51,11 +58,28 @@ class TestTagCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -529,4 +553,22 @@ public boolean matches(String[] argument) { commandLine.handleCommandLine(); verify(mockUntagEntity).handle(); } + + @Test + void testDeleteTagCommandWithoutTagOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.TAG)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TAG, CommandActions.REMOVE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteTag(GravitinoCommandLine.DEFAULT_URL, false, false, "metalake", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_TAG); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java index 50b580eaf72..7fa2e453f32 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java @@ -19,12 +19,20 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.base.Joiner; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateTopic; @@ -35,17 +43,35 @@ import org.apache.gravitino.cli.commands.SetTopicProperty; import org.apache.gravitino.cli.commands.TopicDetails; import org.apache.gravitino.cli.commands.UpdateTopicComment; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestTopicCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -271,4 +297,132 @@ void testRemoveTopicPropertyCommand() { commandLine.handleCommandLine(); verify(mockSetProperties).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testListTopicCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListTopics(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.CATALOG, CommandEntities.SCHEMA))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListTopicCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListTopics(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testTopicDetailsCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTopicDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.CATALOG, CommandEntities.SCHEMA, CommandEntities.TOPIC))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testTopicDetailsCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTopicDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA, CommandEntities.TOPIC))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testTopicDetailsCommandWithoutTopic() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTopicDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "schema", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.TOPIC))); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java index e8a1864b9ff..e8630ce9755 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java @@ -19,12 +19,18 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.AddRoleToUser; @@ -34,17 +40,35 @@ import org.apache.gravitino.cli.commands.RemoveRoleFromUser; import org.apache.gravitino.cli.commands.UserAudit; import org.apache.gravitino.cli.commands.UserDetails; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestUserCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -262,4 +286,22 @@ void testAddRolesToUserCommand() { verify(mockAddFirstRole).handle(); verify(mockAddSecondRole).handle(); } + + @Test + void testDeleteUserWithoutUserOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.USER)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.USER, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteUser(GravitinoCommandLine.DEFAULT_URL, false, false, "metalake_demo", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_USER); + } } From e1acf6314cadea1a991f823e201ded27ee5caadf Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:09:39 +0800 Subject: [PATCH 34/47] [#5930] improvement(CLI): improve unknown tag output. (#5978) ### What changes were proposed in this pull request? Improve output when CLI add an unknown tags. ### Why are the changes needed? Fix: #5930 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? local test --- .../main/java/org/apache/gravitino/cli/commands/TagEntity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java index 55bb4b7f436..d2d1cbbe18f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java @@ -98,7 +98,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = String.join(",", tagsToAdd); + String all = tagsToAdd.length == 0 ? "nothing" : String.join(",", tagsToAdd); System.out.println(entity + " now tagged with " + all); } From 06af45ccdfbf1cfe337b42a36eea7f3765262d06 Mon Sep 17 00:00:00 2001 From: FANNG Date: Thu, 26 Dec 2024 12:19:27 +0800 Subject: [PATCH 35/47] [#5620] feat(fileset): Support credential vending for fileset catalog (#5682) ### What changes were proposed in this pull request? Support credential vending for fileset catalog 1. add `credential-providers` properties for the fileset catalog, schema, and fileset. 2. try to get `credential-providers` from the order of fileset, schema, and catalog. 3. The user could set multi-credential providers ### Why are the changes needed? Fix: #5620 ### Does this PR introduce _any_ user-facing change? will add document after this PR is merged ### How was this patch tested? Add IT and test with local setup Gravitino server --- bundles/aws-bundle/build.gradle.kts | 2 + .../credential/CredentialConstants.java | 1 + .../HadoopCatalogPropertiesMetadata.java | 2 + .../HadoopFilesetPropertiesMetadata.java | 2 + .../HadoopSchemaPropertiesMetadata.java | 2 + .../hadoop/SecureHadoopCatalogOperations.java | 63 +++++-- .../test/FilesetCatalogCredentialIT.java | 160 ++++++++++++++++++ .../org/apache/gravitino/GravitinoEnv.java | 15 +- .../gravitino/catalog/CatalogManager.java | 5 + .../gravitino/catalog/CredentialManager.java | 53 ------ .../gravitino/connector/BaseCatalog.java | 19 +++ .../connector/credential/PathContext.java | 63 +++++++ .../SupportsPathBasedCredentials.java | 43 +++++ .../credential/CatalogCredentialManager.java | 70 ++++++++ .../CredentialOperationDispatcher.java | 124 ++++++++++++++ .../credential/CredentialPrivilege.java | 26 +++ .../gravitino/credential/CredentialUtils.java | 63 ++++++- .../credential/config/CredentialConfig.java | 42 +++++ .../credential/Dummy2CredentialProvider.java | 89 ++++++++++ .../credential/TestCredentialUtils.java | 66 ++++++++ ...he.gravitino.credential.CredentialProvider | 3 +- .../gravitino/server/GravitinoServer.java | 6 +- .../MetadataObjectCredentialOperations.java | 24 ++- ...estMetadataObjectCredentialOperations.java | 13 +- 24 files changed, 867 insertions(+), 89 deletions(-) create mode 100644 clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java delete mode 100644 core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java create mode 100644 core/src/main/java/org/apache/gravitino/connector/credential/PathContext.java create mode 100644 core/src/main/java/org/apache/gravitino/connector/credential/SupportsPathBasedCredentials.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialPrivilege.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java create mode 100644 core/src/test/java/org/apache/gravitino/credential/Dummy2CredentialProvider.java create mode 100644 core/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts index 94c7d1cb2ce..3af5c8b4f38 100644 --- a/bundles/aws-bundle/build.gradle.kts +++ b/bundles/aws-bundle/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { implementation(libs.aws.iam) implementation(libs.aws.policy) implementation(libs.aws.sts) + implementation(libs.commons.lang3) implementation(libs.hadoop3.aws) implementation(project(":catalogs:catalog-common")) { exclude("*") @@ -46,6 +47,7 @@ dependencies { tasks.withType(ShadowJar::class.java) { isZip64 = true configurations = listOf(project.configurations.runtimeClasspath.get()) + relocate("org.apache.commons", "org.apache.gravitino.aws.shaded.org.apache.commons") archiveClassifier.set("") } diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index 29f9241c890..d2753f24b5e 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -21,6 +21,7 @@ public class CredentialConstants { public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; + public static final String CREDENTIAL_PROVIDERS = "credential-providers"; public static final String S3_TOKEN_CREDENTIAL_PROVIDER = "s3-token"; public static final String S3_TOKEN_EXPIRE_IN_SECS = "s3-token-expire-in-secs"; diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 397e13aa4af..22cf0d5b2cd 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -27,6 +27,7 @@ import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; +import org.apache.gravitino.credential.config.CredentialConfig; public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetadata { @@ -84,6 +85,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada // The following two are about authentication. .putAll(KERBEROS_PROPERTY_ENTRIES) .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES) + .putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES) .build(); @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java index 250a48d292f..84862dd0941 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java @@ -24,6 +24,7 @@ import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import org.apache.gravitino.connector.BasePropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; +import org.apache.gravitino.credential.config.CredentialConfig; public class HadoopFilesetPropertiesMetadata extends BasePropertiesMetadata { @@ -32,6 +33,7 @@ protected Map> specificPropertyEntries() { ImmutableMap.Builder> builder = ImmutableMap.builder(); builder.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES); builder.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES); + builder.putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES); return builder.build(); } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java index 8892433ac6c..9028cc48f3b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java @@ -24,6 +24,7 @@ import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import org.apache.gravitino.connector.BasePropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; +import org.apache.gravitino.credential.config.CredentialConfig; public class HadoopSchemaPropertiesMetadata extends BasePropertiesMetadata { @@ -49,6 +50,7 @@ public class HadoopSchemaPropertiesMetadata extends BasePropertiesMetadata { false /* hidden */)) .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES) .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES) + .putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES) .build(); @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java index 2180e45d423..7ae10805b5b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java @@ -20,11 +20,16 @@ package org.apache.gravitino.catalog.hadoop; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import javax.security.auth.Subject; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityStore; @@ -38,6 +43,9 @@ import org.apache.gravitino.connector.CatalogOperations; import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.SupportsSchemas; +import org.apache.gravitino.connector.credential.PathContext; +import org.apache.gravitino.connector.credential.SupportsPathBasedCredentials; +import org.apache.gravitino.credential.CredentialUtils; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchEntityException; @@ -50,13 +58,14 @@ import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.meta.FilesetEntity; import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.PrincipalUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @SuppressWarnings("removal") public class SecureHadoopCatalogOperations - implements CatalogOperations, SupportsSchemas, FilesetCatalog { + implements CatalogOperations, SupportsSchemas, FilesetCatalog, SupportsPathBasedCredentials { public static final Logger LOG = LoggerFactory.getLogger(SecureHadoopCatalogOperations.class); @@ -66,6 +75,8 @@ public class SecureHadoopCatalogOperations private UserContext catalogUserContext; + private Map catalogProperties; + public SecureHadoopCatalogOperations() { this.hadoopCatalogOperations = new HadoopCatalogOperations(); } @@ -74,6 +85,20 @@ public SecureHadoopCatalogOperations(EntityStore store) { this.hadoopCatalogOperations = new HadoopCatalogOperations(store); } + @Override + public void initialize( + Map config, CatalogInfo info, HasPropertyMetadata propertiesMetadata) + throws RuntimeException { + hadoopCatalogOperations.initialize(config, info, propertiesMetadata); + this.catalogUserContext = + UserContext.getUserContext( + NameIdentifier.of(info.namespace(), info.name()), + config, + hadoopCatalogOperations.getHadoopConf(), + info); + this.catalogProperties = info.properties(); + } + @VisibleForTesting public HadoopCatalogOperations getBaseHadoopCatalogOperations() { return hadoopCatalogOperations; @@ -163,19 +188,6 @@ public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmpty } } - @Override - public void initialize( - Map config, CatalogInfo info, HasPropertyMetadata propertiesMetadata) - throws RuntimeException { - hadoopCatalogOperations.initialize(config, info, propertiesMetadata); - catalogUserContext = - UserContext.getUserContext( - NameIdentifier.of(info.namespace(), info.name()), - config, - hadoopCatalogOperations.getHadoopConf(), - info); - } - @Override public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) throws NoSuchFilesetException, IllegalArgumentException { @@ -245,6 +257,29 @@ public void testConnection( hadoopCatalogOperations.testConnection(catalogIdent, type, provider, comment, properties); } + @Override + public List getPathContext(NameIdentifier filesetIdentifier) { + Fileset fileset = loadFileset(filesetIdentifier); + String path = fileset.storageLocation(); + Preconditions.checkState( + StringUtils.isNotBlank(path), "The location of fileset should not be empty."); + + Set providers = + CredentialUtils.getCredentialProvidersByOrder( + () -> fileset.properties(), + () -> { + Namespace namespace = filesetIdentifier.namespace(); + NameIdentifier schemaIdentifier = + NameIdentifierUtil.ofSchema( + namespace.level(0), namespace.level(1), namespace.level(2)); + return loadSchema(schemaIdentifier).properties(); + }, + () -> catalogProperties); + return providers.stream() + .map(provider -> new PathContext(path, provider)) + .collect(Collectors.toList()); + } + /** * Add the user to the subject so that we can get the last user in the subject. Hadoop catalog * uses this method to pass api user from the client side, so that we can get the user in the diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java new file mode 100644 index 00000000000..94239fef28f --- /dev/null +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.filesystem.hadoop.integration.test; + +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.S3SecretKeyCredential; +import org.apache.gravitino.credential.S3TokenCredential; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.storage.S3Properties; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") +public class FilesetCatalogCredentialIT extends BaseIT { + + private static final Logger LOG = LoggerFactory.getLogger(FilesetCatalogCredentialIT.class); + + public static final String BUCKET_NAME = System.getenv("S3_BUCKET_NAME"); + public static final String S3_ACCESS_KEY = System.getenv("S3_ACCESS_KEY_ID"); + public static final String S3_SECRET_KEY = System.getenv("S3_SECRET_ACCESS_KEY"); + public static final String S3_ROLE_ARN = System.getenv("S3_ROLE_ARN"); + + private String metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); + private String catalogName = GravitinoITUtils.genRandomName("catalog"); + private String schemaName = GravitinoITUtils.genRandomName("schema"); + private GravitinoMetalake metalake; + + @BeforeAll + public void startIntegrationTest() { + // Do nothing + } + + @BeforeAll + public void startUp() throws Exception { + copyBundleJarsToHadoop("aws-bundle"); + // Need to download jars to gravitino server + super.startIntegrationTest(); + + metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); + catalogName = GravitinoITUtils.genRandomName("catalog"); + schemaName = GravitinoITUtils.genRandomName("schema"); + + Assertions.assertFalse(client.metalakeExists(metalakeName)); + metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); + Assertions.assertTrue(client.metalakeExists(metalakeName)); + + Map properties = Maps.newHashMap(); + properties.put(FILESYSTEM_PROVIDERS, "s3"); + properties.put( + CredentialConstants.CREDENTIAL_PROVIDERS, + S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE + + "," + + S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE); + properties.put( + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE); + properties.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, S3_ACCESS_KEY); + properties.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, S3_SECRET_KEY); + properties.put(S3Properties.GRAVITINO_S3_ENDPOINT, "s3.ap-southeast-2.amazonaws.com"); + properties.put(S3Properties.GRAVITINO_S3_REGION, "ap-southeast-2"); + properties.put(S3Properties.GRAVITINO_S3_ROLE_ARN, S3_ROLE_ARN); + + Catalog catalog = + metalake.createCatalog( + catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", properties); + Assertions.assertTrue(metalake.catalogExists(catalogName)); + + catalog.asSchemas().createSchema(schemaName, "schema comment", properties); + Assertions.assertTrue(catalog.asSchemas().schemaExists(schemaName)); + } + + @AfterAll + public void tearDown() throws IOException { + Catalog catalog = metalake.loadCatalog(catalogName); + catalog.asSchemas().dropSchema(schemaName, true); + metalake.dropCatalog(catalogName, true); + client.dropMetalake(metalakeName, true); + + if (client != null) { + client.close(); + client = null; + } + + try { + closer.close(); + } catch (Exception e) { + LOG.error("Exception in closing CloseableGroup", e); + } + } + + protected String genStorageLocation(String fileset) { + return String.format("s3a://%s/%s", BUCKET_NAME, fileset); + } + + @Test + void testGetCatalogCredential() { + Catalog catalog = metalake.loadCatalog(catalogName); + Credential[] credentials = catalog.supportsCredentials().getCredentials(); + Assertions.assertEquals(1, credentials.length); + Assertions.assertTrue(credentials[0] instanceof S3SecretKeyCredential); + } + + @Test + void testGetFilesetCredential() { + String filesetName = GravitinoITUtils.genRandomName("test_fileset_credential"); + NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); + Catalog catalog = metalake.loadCatalog(catalogName); + String storageLocation = genStorageLocation(filesetName); + catalog + .asFilesetCatalog() + .createFileset( + filesetIdent, + "fileset comment", + Fileset.Type.MANAGED, + storageLocation, + ImmutableMap.of( + CredentialConstants.CREDENTIAL_PROVIDERS, + S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE)); + + Fileset fileset = catalog.asFilesetCatalog().loadFileset(filesetIdent); + Credential[] credentials = fileset.supportsCredentials().getCredentials(); + Assertions.assertEquals(1, credentials.length); + Assertions.assertTrue(credentials[0] instanceof S3TokenCredential); + } +} diff --git a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java index 96c60b834fc..57f04a0cfbf 100644 --- a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java +++ b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java @@ -28,7 +28,6 @@ import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.catalog.CatalogManager; import org.apache.gravitino.catalog.CatalogNormalizeDispatcher; -import org.apache.gravitino.catalog.CredentialManager; import org.apache.gravitino.catalog.FilesetDispatcher; import org.apache.gravitino.catalog.FilesetNormalizeDispatcher; import org.apache.gravitino.catalog.FilesetOperationDispatcher; @@ -47,6 +46,7 @@ import org.apache.gravitino.catalog.TopicDispatcher; import org.apache.gravitino.catalog.TopicNormalizeDispatcher; import org.apache.gravitino.catalog.TopicOperationDispatcher; +import org.apache.gravitino.credential.CredentialOperationDispatcher; import org.apache.gravitino.hook.AccessControlHookDispatcher; import org.apache.gravitino.hook.CatalogHookDispatcher; import org.apache.gravitino.hook.FilesetHookDispatcher; @@ -108,7 +108,7 @@ public class GravitinoEnv { private MetalakeDispatcher metalakeDispatcher; - private CredentialManager credentialManager; + private CredentialOperationDispatcher credentialOperationDispatcher; private TagDispatcher tagDispatcher; @@ -264,12 +264,12 @@ public MetalakeDispatcher metalakeDispatcher() { } /** - * Get the {@link CredentialManager} associated with the Gravitino environment. + * Get the {@link CredentialOperationDispatcher} associated with the Gravitino environment. * - * @return The {@link CredentialManager} instance. + * @return The {@link CredentialOperationDispatcher} instance. */ - public CredentialManager credentialManager() { - return credentialManager; + public CredentialOperationDispatcher credentialOperationDispatcher() { + return credentialOperationDispatcher; } /** @@ -432,7 +432,8 @@ private void initGravitinoServerComponents() { new CatalogNormalizeDispatcher(catalogHookDispatcher); this.catalogDispatcher = new CatalogEventDispatcher(eventBus, catalogNormalizeDispatcher); - this.credentialManager = new CredentialManager(catalogManager, entityStore, idGenerator); + this.credentialOperationDispatcher = + new CredentialOperationDispatcher(catalogManager, entityStore, idGenerator); SchemaOperationDispatcher schemaOperationDispatcher = new SchemaOperationDispatcher(catalogManager, entityStore, idGenerator); diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index 4a46952f87e..1e9c1d9d94f 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -126,6 +126,7 @@ public static void checkCatalogInUse(EntityStore store, NameIdentifier ident) /** Wrapper class for a catalog instance and its class loader. */ public static class CatalogWrapper { + private BaseCatalog catalog; private IsolatedClassLoader classLoader; @@ -169,6 +170,10 @@ public R doWithFilesetOps(ThrowableFunction fn) throws Ex }); } + public R doWithCredentialOps(ThrowableFunction fn) throws Exception { + return classLoader.withClassLoader(cl -> fn.apply(catalog)); + } + public R doWithTopicOps(ThrowableFunction fn) throws Exception { return classLoader.withClassLoader( cl -> { diff --git a/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java b/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java deleted file mode 100644 index 808fc96fb0a..00000000000 --- a/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.catalog; - -import java.util.List; -import org.apache.commons.lang3.NotImplementedException; -import org.apache.gravitino.EntityStore; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.connector.BaseCatalog; -import org.apache.gravitino.credential.Credential; -import org.apache.gravitino.exceptions.NoSuchCatalogException; -import org.apache.gravitino.storage.IdGenerator; -import org.apache.gravitino.utils.NameIdentifierUtil; - -/** Get credentials with the specific catalog classloader. */ -public class CredentialManager extends OperationDispatcher { - - public CredentialManager( - CatalogManager catalogManager, EntityStore store, IdGenerator idGenerator) { - super(catalogManager, store, idGenerator); - } - - public List getCredentials(NameIdentifier identifier) { - return doWithCatalog( - NameIdentifierUtil.getCatalogIdentifier(identifier), - c -> getCredentials(c.catalog(), identifier), - NoSuchCatalogException.class); - } - - private List getCredentials(BaseCatalog catalog, NameIdentifier identifier) { - throw new NotImplementedException( - String.format( - "Load credentials is not implemented for catalog: %s, identifier: %s", - catalog.name(), identifier)); - } -} diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java index 218c2a428b3..14b1912b4d6 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java @@ -31,6 +31,7 @@ import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; import org.apache.gravitino.connector.capability.Capability; +import org.apache.gravitino.credential.CatalogCredentialManager; import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.utils.IsolatedClassLoader; import org.slf4j.Logger; @@ -51,6 +52,7 @@ @Evolving public abstract class BaseCatalog implements Catalog, CatalogProvider, HasPropertyMetadata, Closeable { + private static final Logger LOG = LoggerFactory.getLogger(BaseCatalog.class); // This variable is used as a key in properties of catalogs to inject custom operation to @@ -72,6 +74,8 @@ public abstract class BaseCatalog private volatile Map properties; + private volatile CatalogCredentialManager catalogCredentialManager; + private static String ENTITY_IS_NOT_SET = "entity is not set"; // Any Gravitino configuration that starts with this prefix will be trim and passed to the @@ -225,6 +229,10 @@ public void close() throws IOException { authorizationPlugin.close(); authorizationPlugin = null; } + if (catalogCredentialManager != null) { + catalogCredentialManager.close(); + catalogCredentialManager = null; + } } public Capability capability() { @@ -239,6 +247,17 @@ public Capability capability() { return capability; } + public CatalogCredentialManager catalogCredentialManager() { + if (catalogCredentialManager == null) { + synchronized (this) { + if (catalogCredentialManager == null) { + this.catalogCredentialManager = new CatalogCredentialManager(name(), properties()); + } + } + } + return catalogCredentialManager; + } + private CatalogOperations createOps(Map conf) { String customCatalogOperationClass = conf.get(CATALOG_OPERATION_IMPL); return Optional.ofNullable(customCatalogOperationClass) diff --git a/core/src/main/java/org/apache/gravitino/connector/credential/PathContext.java b/core/src/main/java/org/apache/gravitino/connector/credential/PathContext.java new file mode 100644 index 00000000000..5c520d6bfda --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/connector/credential/PathContext.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.connector.credential; + +import org.apache.gravitino.annotation.DeveloperApi; + +/** + * The {@code PathContext} class represents the path and its associated credential type to generate + * a credential for {@link org.apache.gravitino.credential.CredentialOperationDispatcher}. + */ +@DeveloperApi +public class PathContext { + + private final String path; + + private final String credentialType; + + /** + * Constructs a new {@code PathContext} instance with the given path and credential type. + * + * @param path The path string. + * @param credentialType The type of the credential. + */ + public PathContext(String path, String credentialType) { + this.path = path; + this.credentialType = credentialType; + } + + /** + * Gets the path string. + * + * @return The path associated with this instance. + */ + public String path() { + return path; + } + + /** + * Gets the credential type. + * + * @return The credential type associated with this instance. + */ + public String credentialType() { + return credentialType; + } +} diff --git a/core/src/main/java/org/apache/gravitino/connector/credential/SupportsPathBasedCredentials.java b/core/src/main/java/org/apache/gravitino/connector/credential/SupportsPathBasedCredentials.java new file mode 100644 index 00000000000..93e08a39069 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/connector/credential/SupportsPathBasedCredentials.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.connector.credential; + +import java.util.List; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.annotation.DeveloperApi; + +/** The catalog operation should implement this interface to generate the path based credentials. */ +@DeveloperApi +public interface SupportsPathBasedCredentials { + + /** + * Get {@link PathContext} lists. + * + *

In most cases there will be only one element in the list. For catalogs which support multi + * locations like fileset, there may be multiple elements. + * + *

The name identifier is the identifier of the resource like fileset, table, etc. not include + * metalake, catalog, schema. + * + * @param nameIdentifier, The identifier for fileset, table, etc. + * @return A list of {@link PathContext} + */ + List getPathContext(NameIdentifier nameIdentifier); +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java new file mode 100644 index 00000000000..2fe6fedccd9 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manage lifetime of the credential provider in one catalog, dispatch credential request to the + * corresponding credential provider. + */ +public class CatalogCredentialManager implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(CatalogCredentialManager.class); + + private final String catalogName; + private final Map credentialProviders; + + public CatalogCredentialManager(String catalogName, Map catalogProperties) { + this.catalogName = catalogName; + this.credentialProviders = CredentialUtils.loadCredentialProviders(catalogProperties); + } + + public Credential getCredential(String credentialType, CredentialContext context) { + // todo: add credential cache + Preconditions.checkState( + credentialProviders.containsKey(credentialType), + String.format("Credential %s not found", credentialType)); + return credentialProviders.get(credentialType).getCredential(context); + } + + @Override + public void close() { + credentialProviders + .values() + .forEach( + credentialProvider -> { + try { + credentialProvider.close(); + } catch (IOException e) { + LOG.warn( + "Close credential provider failed, catalog: {}, credential provider: {}", + catalogName, + credentialProvider.credentialType(), + e); + } + }); + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java new file mode 100644 index 00000000000..2ec76aeb4ad --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import javax.ws.rs.NotAuthorizedException; +import javax.ws.rs.NotSupportedException; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.catalog.CatalogManager; +import org.apache.gravitino.catalog.OperationDispatcher; +import org.apache.gravitino.connector.BaseCatalog; +import org.apache.gravitino.connector.credential.PathContext; +import org.apache.gravitino.connector.credential.SupportsPathBasedCredentials; +import org.apache.gravitino.exceptions.NoSuchCatalogException; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; + +/** Get credentials with the specific catalog classloader. */ +public class CredentialOperationDispatcher extends OperationDispatcher { + + public CredentialOperationDispatcher( + CatalogManager catalogManager, EntityStore store, IdGenerator idGenerator) { + super(catalogManager, store, idGenerator); + } + + public List getCredentials(NameIdentifier identifier) { + CredentialPrivilege privilege = + getCredentialPrivilege(PrincipalUtils.getCurrentUserName(), identifier); + return doWithCatalog( + NameIdentifierUtil.getCatalogIdentifier(identifier), + catalogWrapper -> + catalogWrapper.doWithCredentialOps( + baseCatalog -> getCredentials(baseCatalog, identifier, privilege)), + NoSuchCatalogException.class); + } + + private List getCredentials( + BaseCatalog baseCatalog, NameIdentifier nameIdentifier, CredentialPrivilege privilege) { + Map contexts = + getCredentialContexts(baseCatalog, nameIdentifier, privilege); + return contexts.entrySet().stream() + .map( + entry -> + baseCatalog + .catalogCredentialManager() + .getCredential(entry.getKey(), entry.getValue())) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private Map getCredentialContexts( + BaseCatalog baseCatalog, NameIdentifier nameIdentifier, CredentialPrivilege privilege) { + if (nameIdentifier.equals(NameIdentifierUtil.getCatalogIdentifier(nameIdentifier))) { + return getCatalogCredentialContexts(baseCatalog.properties()); + } + + if (baseCatalog.ops() instanceof SupportsPathBasedCredentials) { + List pathContexts = + ((SupportsPathBasedCredentials) baseCatalog.ops()).getPathContext(nameIdentifier); + return getPathBasedCredentialContexts(privilege, pathContexts); + } + throw new NotSupportedException( + String.format("Catalog %s doesn't support generate credentials", baseCatalog.name())); + } + + private Map getCatalogCredentialContexts( + Map catalogProperties) { + CatalogCredentialContext context = + new CatalogCredentialContext(PrincipalUtils.getCurrentUserName()); + Set providers = CredentialUtils.getCredentialProvidersByOrder(() -> catalogProperties); + return providers.stream().collect(Collectors.toMap(provider -> provider, provider -> context)); + } + + public static Map getPathBasedCredentialContexts( + CredentialPrivilege privilege, List pathContexts) { + return pathContexts.stream() + .collect( + Collectors.toMap( + pathContext -> pathContext.credentialType(), + pathContext -> { + String path = pathContext.path(); + Set writePaths = new HashSet<>(); + Set readPaths = new HashSet<>(); + if (CredentialPrivilege.WRITE.equals(privilege)) { + writePaths.add(path); + } else { + readPaths.add(path); + } + return new PathBasedCredentialContext( + PrincipalUtils.getCurrentUserName(), writePaths, readPaths); + })); + } + + @SuppressWarnings("UnusedVariable") + private CredentialPrivilege getCredentialPrivilege(String user, NameIdentifier identifier) + throws NotAuthorizedException { + // TODO: will implement in another PR + return CredentialPrivilege.WRITE; + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialPrivilege.java b/core/src/main/java/org/apache/gravitino/credential/CredentialPrivilege.java new file mode 100644 index 00000000000..3ff77cd3e8f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialPrivilege.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +/** Represents the privilege to get credential from credential providers. */ +public enum CredentialPrivilege { + READ, + WRITE, +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java index 09439d58ae8..9a202ec9747 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java @@ -19,14 +19,75 @@ package org.apache.gravitino.credential; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.gravitino.utils.PrincipalUtils; public class CredentialUtils { + + private static final Splitter splitter = Splitter.on(","); + public static Credential vendCredential(CredentialProvider credentialProvider, String[] path) { PathBasedCredentialContext pathBasedCredentialContext = new PathBasedCredentialContext( - PrincipalUtils.getCurrentUserName(), ImmutableSet.copyOf(path), ImmutableSet.of()); + PrincipalUtils.getCurrentUserName(), ImmutableSet.copyOf(path), Collections.emptySet()); return credentialProvider.getCredential(pathBasedCredentialContext); } + + public static Map loadCredentialProviders( + Map catalogProperties) { + Set credentialProviders = + CredentialUtils.getCredentialProvidersByOrder(() -> catalogProperties); + + return credentialProviders.stream() + .collect( + Collectors.toMap( + String::toString, + credentialType -> + CredentialProviderFactory.create(credentialType, catalogProperties))); + } + + /** + * Get Credential providers from properties supplier. + * + *

If there are multiple properties suppliers, will try to get the credential providers in the + * input order. + * + * @param propertiesSuppliers The properties suppliers. + * @return A set of credential providers. + */ + public static Set getCredentialProvidersByOrder( + Supplier>... propertiesSuppliers) { + + for (Supplier> supplier : propertiesSuppliers) { + Map properties = supplier.get(); + Set providers = getCredentialProvidersFromProperties(properties); + if (!providers.isEmpty()) { + return providers; + } + } + + return Collections.emptySet(); + } + + private static Set getCredentialProvidersFromProperties(Map properties) { + if (properties == null) { + return Collections.emptySet(); + } + + String providers = properties.get(CredentialConstants.CREDENTIAL_PROVIDERS); + if (providers == null) { + return Collections.emptySet(); + } + return splitter + .trimResults() + .omitEmptyStrings() + .splitToStream(providers) + .collect(Collectors.toSet()); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java new file mode 100644 index 00000000000..d8823417cda --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential.config; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.connector.PropertyEntry; +import org.apache.gravitino.credential.CredentialConstants; + +public class CredentialConfig { + + public static final Map> CREDENTIAL_PROPERTY_ENTRIES = + new ImmutableMap.Builder>() + .put( + CredentialConstants.CREDENTIAL_PROVIDERS, + PropertyEntry.booleanPropertyEntry( + CredentialConstants.CREDENTIAL_PROVIDERS, + "Credential providers for the Gravitino catalog, schema, fileset, table, etc.", + false /* required */, + false /* immutable */, + null /* default value */, + false /* hidden */, + false /* reserved */)) + .build(); +} diff --git a/core/src/test/java/org/apache/gravitino/credential/Dummy2CredentialProvider.java b/core/src/test/java/org/apache/gravitino/credential/Dummy2CredentialProvider.java new file mode 100644 index 00000000000..63f63d61d0b --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/credential/Dummy2CredentialProvider.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.Set; +import javax.ws.rs.NotSupportedException; +import lombok.Getter; + +public class Dummy2CredentialProvider implements CredentialProvider { + Map properties; + static final String CREDENTIAL_TYPE = "dummy2"; + + @Override + public void initialize(Map properties) { + this.properties = properties; + } + + @Override + public void close() {} + + @Override + public String credentialType() { + return CREDENTIAL_TYPE; + } + + @Override + public Credential getCredential(CredentialContext context) { + Preconditions.checkArgument( + context instanceof PathBasedCredentialContext + || context instanceof CatalogCredentialContext, + "Doesn't support context: " + context.getClass().getSimpleName()); + if (context instanceof PathBasedCredentialContext) { + return new Dummy2Credential((PathBasedCredentialContext) context); + } + return null; + } + + public static class Dummy2Credential implements Credential { + + @Getter private Set writeLocations; + @Getter private Set readLocations; + + public Dummy2Credential(PathBasedCredentialContext locationContext) { + this.writeLocations = locationContext.getWritePaths(); + this.readLocations = locationContext.getReadPaths(); + } + + @Override + public String credentialType() { + return Dummy2CredentialProvider.CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return 0; + } + + @Override + public Map credentialInfo() { + return ImmutableMap.of( + "writeLocation", writeLocations.toString(), "readLocation", readLocations.toString()); + } + + @Override + public void initialize(Map credentialInfo, long expireTimeInMs) { + throw new NotSupportedException(); + } + } +} diff --git a/core/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java b/core/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java new file mode 100644 index 00000000000..c31affdc157 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import org.testcontainers.shaded.com.google.common.collect.ImmutableSet; + +public class TestCredentialUtils { + + @Test + void testLoadCredentialProviders() { + Map catalogProperties = + ImmutableMap.of( + CredentialConstants.CREDENTIAL_PROVIDERS, + DummyCredentialProvider.CREDENTIAL_TYPE + + "," + + Dummy2CredentialProvider.CREDENTIAL_TYPE); + Map providers = + CredentialUtils.loadCredentialProviders(catalogProperties); + Assertions.assertTrue(providers.size() == 2); + + Assertions.assertTrue(providers.containsKey(DummyCredentialProvider.CREDENTIAL_TYPE)); + Assertions.assertTrue( + DummyCredentialProvider.CREDENTIAL_TYPE.equals( + providers.get(DummyCredentialProvider.CREDENTIAL_TYPE).credentialType())); + Assertions.assertTrue(providers.containsKey(Dummy2CredentialProvider.CREDENTIAL_TYPE)); + Assertions.assertTrue( + Dummy2CredentialProvider.CREDENTIAL_TYPE.equals( + providers.get(Dummy2CredentialProvider.CREDENTIAL_TYPE).credentialType())); + } + + @Test + void testGetCredentialProviders() { + Map filesetProperties = ImmutableMap.of(); + Map schemaProperties = + ImmutableMap.of(CredentialConstants.CREDENTIAL_PROVIDERS, "a,b"); + Map catalogProperties = + ImmutableMap.of(CredentialConstants.CREDENTIAL_PROVIDERS, "a,b,c"); + + Set credentialProviders = + CredentialUtils.getCredentialProvidersByOrder( + () -> filesetProperties, () -> schemaProperties, () -> catalogProperties); + Assertions.assertEquals(credentialProviders, ImmutableSet.of("a", "b")); + } +} diff --git a/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider index cbdbff0bee9..6e1fdde4bdb 100644 --- a/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider +++ b/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -16,4 +16,5 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.credential.DummyCredentialProvider \ No newline at end of file +org.apache.gravitino.credential.DummyCredentialProvider +org.apache.gravitino.credential.Dummy2CredentialProvider diff --git a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java index 16a2096f328..63e53aefd59 100644 --- a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java @@ -26,12 +26,12 @@ import org.apache.gravitino.Configs; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.catalog.CatalogDispatcher; -import org.apache.gravitino.catalog.CredentialManager; import org.apache.gravitino.catalog.FilesetDispatcher; import org.apache.gravitino.catalog.PartitionDispatcher; import org.apache.gravitino.catalog.SchemaDispatcher; import org.apache.gravitino.catalog.TableDispatcher; import org.apache.gravitino.catalog.TopicDispatcher; +import org.apache.gravitino.credential.CredentialOperationDispatcher; import org.apache.gravitino.metalake.MetalakeDispatcher; import org.apache.gravitino.metrics.MetricsSystem; import org.apache.gravitino.metrics.source.MetricsSource; @@ -115,7 +115,9 @@ protected void configure() { bind(gravitinoEnv.filesetDispatcher()).to(FilesetDispatcher.class).ranked(1); bind(gravitinoEnv.topicDispatcher()).to(TopicDispatcher.class).ranked(1); bind(gravitinoEnv.tagDispatcher()).to(TagDispatcher.class).ranked(1); - bind(gravitinoEnv.credentialManager()).to(CredentialManager.class).ranked(1); + bind(gravitinoEnv.credentialOperationDispatcher()) + .to(CredentialOperationDispatcher.class) + .ranked(1); } }); register(JsonProcessingExceptionMapper.class); diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java index 7c6ea4a8eb7..1046bbba1a5 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java @@ -21,11 +21,14 @@ import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Locale; +import java.util.Set; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; +import javax.ws.rs.NotSupportedException; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -34,8 +37,8 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.catalog.CredentialManager; import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialOperationDispatcher; import org.apache.gravitino.dto.credential.CredentialDTO; import org.apache.gravitino.dto.responses.CredentialResponse; import org.apache.gravitino.dto.util.DTOConverters; @@ -51,15 +54,18 @@ public class MetadataObjectCredentialOperations { private static final Logger LOG = LoggerFactory.getLogger(MetadataObjectCredentialOperations.class); - private CredentialManager credentialManager; + private static final Set supportsCredentialMetadataTypes = + ImmutableSet.of(MetadataObject.Type.CATALOG, MetadataObject.Type.FILESET); + + private CredentialOperationDispatcher credentialOperationDispatcher; @SuppressWarnings("unused") @Context private HttpServletRequest httpRequest; @Inject - public MetadataObjectCredentialOperations(CredentialManager dispatcher) { - this.credentialManager = dispatcher; + public MetadataObjectCredentialOperations(CredentialOperationDispatcher dispatcher) { + this.credentialOperationDispatcher = dispatcher; } @GET @@ -83,9 +89,13 @@ public Response getCredentials( MetadataObject object = MetadataObjects.parse( fullName, MetadataObject.Type.valueOf(type.toUpperCase(Locale.ROOT))); + if (!supportsCredentialOperations(object)) { + throw new NotSupportedException( + "Doesn't support credential operations for metadata object type"); + } NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, object); - List credentials = credentialManager.getCredentials(identifier); + List credentials = credentialOperationDispatcher.getCredentials(identifier); if (credentials == null) { return Utils.ok(new CredentialResponse(new CredentialDTO[0])); } @@ -97,4 +107,8 @@ public Response getCredentials( return ExceptionHandlers.handleCredentialException(OperationType.GET, fullName, e); } } + + private static boolean supportsCredentialOperations(MetadataObject metadataObject) { + return supportsCredentialMetadataTypes.contains(metadataObject.type()); + } } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java index 1ac5d38135d..464ccd86984 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java @@ -31,8 +31,8 @@ import javax.ws.rs.core.Response; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; -import org.apache.gravitino.catalog.CredentialManager; import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialOperationDispatcher; import org.apache.gravitino.credential.S3SecretKeyCredential; import org.apache.gravitino.dto.responses.CredentialResponse; import org.apache.gravitino.dto.responses.ErrorConstants; @@ -59,7 +59,8 @@ public HttpServletRequest get() { } } - private CredentialManager credentialManager = mock(CredentialManager.class); + private CredentialOperationDispatcher credentialOperationDispatcher = + mock(CredentialOperationDispatcher.class); private String metalake = "test_metalake"; @@ -78,7 +79,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(credentialManager).to(CredentialManager.class).ranked(2); + bind(credentialOperationDispatcher).to(CredentialOperationDispatcher.class).ranked(2); bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); } }); @@ -101,7 +102,7 @@ private void testGetCredentialsForObject(MetadataObject metadataObject) { S3SecretKeyCredential credential = new S3SecretKeyCredential("access-id", "secret-key"); // Test return one credential - when(credentialManager.getCredentials(any())).thenReturn(Arrays.asList(credential)); + when(credentialOperationDispatcher.getCredentials(any())).thenReturn(Arrays.asList(credential)); Response response = target(basePath(metalake)) .path(metadataObject.type().toString()) @@ -123,7 +124,7 @@ private void testGetCredentialsForObject(MetadataObject metadataObject) { Assertions.assertEquals(0, credentialToTest.expireTimeInMs()); // Test doesn't return credential - when(credentialManager.getCredentials(any())).thenReturn(null); + when(credentialOperationDispatcher.getCredentials(any())).thenReturn(null); response = target(basePath(metalake)) .path(metadataObject.type().toString()) @@ -140,7 +141,7 @@ private void testGetCredentialsForObject(MetadataObject metadataObject) { // Test throws NoSuchCredentialException doThrow(new NoSuchCredentialException("mock error")) - .when(credentialManager) + .when(credentialOperationDispatcher) .getCredentials(any()); response = target(basePath(metalake)) From 6aa714db8df8dd420cd05820141f73d846056fa7 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Thu, 26 Dec 2024 15:30:55 +0800 Subject: [PATCH 36/47] [#5817] core(feat): Add server-side REST APIs for model management (#5948) ### What changes were proposed in this pull request? This PR adds the server-side REST endpoint for model management. ### Why are the changes needed? This is a part of model management for Gravitino. Fix: #5817 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add UTs for this PR. --- .../apache/gravitino/model/ModelCatalog.java | 8 +- .../catalog/model/ModelCatalogImpl.java | 3 +- .../org.apache.gravitino.CatalogProvider | 2 +- .../apache/gravitino/dto/model/ModelDTO.java | 163 ++++ .../gravitino/dto/model/ModelVersionDTO.java | 184 ++++ .../dto/requests/CatalogCreateRequest.java | 31 +- .../dto/requests/ModelRegisterRequest.java | 54 ++ .../dto/requests/ModelVersionLinkRequest.java | 67 ++ .../dto/responses/ModelResponse.java | 59 ++ .../responses/ModelVersionListResponse.java | 57 ++ .../dto/responses/ModelVersionResponse.java | 59 ++ .../gravitino/dto/util/DTOConverters.java | 37 + .../gravitino/dto/model/TestModelDTO.java | 81 ++ .../dto/model/TestModelVersionDTO.java | 133 +++ .../requests/TestCatalogCreateRequest.java | 15 +- .../requests/TestModelRegisterRequest.java | 47 + .../requests/TestModelVersionLinkRequest.java | 75 ++ .../dto/responses/TestResponses.java | 76 ++ docs/open-api/models.yaml | 561 ++++++++++++ docs/open-api/openapi.yaml | 23 + .../gravitino/server/GravitinoServer.java | 2 + .../server/web/rest/ExceptionHandlers.java | 45 + .../server/web/rest/ModelOperations.java | 411 +++++++++ .../server/web/rest/OperationType.java | 3 + .../server/web/rest/TestModelOperations.java | 843 ++++++++++++++++++ 25 files changed, 3006 insertions(+), 33 deletions(-) create mode 100644 common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java create mode 100644 docs/open-api/models.yaml create mode 100644 server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java create mode 100644 server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java diff --git a/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java b/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java index cea2e94e3c7..3fb39c18aea 100644 --- a/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java +++ b/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java @@ -93,7 +93,10 @@ Model registerModel(NameIdentifier ident, String comment, Map pr * * @param ident The name identifier of the model. * @param uri The model artifact URI. - * @param aliases The aliases of the model version. The alias are optional and can be empty. + * @param aliases The aliases of the model version. The aliases should be unique in this model, + * otherwise the {@link ModelVersionAliasesAlreadyExistException} will be thrown. The aliases + * are optional and can be empty. Also, be aware that the alias cannot be a number or a number + * string. * @param comment The comment of the model. The comment is optional and can be null. * @param properties The properties of the model. The properties are optional and can be null or * empty. @@ -198,7 +201,8 @@ default boolean modelVersionExists(NameIdentifier ident, String alias) { * @param uri The URI of the model version artifact. * @param aliases The aliases of the model version. The aliases should be unique in this model, * otherwise the {@link ModelVersionAliasesAlreadyExistException} will be thrown. The aliases - * are optional and can be empty. + * are optional and can be empty. Also, be aware that the alias cannot be a number or a number + * string. * @param comment The comment of the model version. The comment is optional and can be null. * @param properties The properties of the model version. The properties are optional and can be * null or empty. diff --git a/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java index 5b90eab7265..545f6482a3f 100644 --- a/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java +++ b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java @@ -19,7 +19,6 @@ package org.apache.gravitino.catalog.model; import java.util.Map; -import org.apache.gravitino.CatalogProvider; import org.apache.gravitino.EntityStore; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.connector.BaseCatalog; @@ -40,7 +39,7 @@ public class ModelCatalogImpl extends BaseCatalog { @Override public String shortName() { - return CatalogProvider.shortNameForManagedCatalog(super.type()); + return "model"; } @Override diff --git a/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider b/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider index 37c682aa745..e43f995ea7d 100644 --- a/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider +++ b/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider @@ -16,4 +16,4 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.catalog.model.ModelCatalog +org.apache.gravitino.catalog.model.ModelCatalogImpl diff --git a/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java b/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java new file mode 100644 index 00000000000..44688399335 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.model.Model; + +/** Represents a model DTO (Data Transfer Object). */ +@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class ModelDTO implements Model { + + @JsonProperty("name") + private String name; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @JsonProperty("latestVersion") + private int latestVersion; + + @JsonProperty("audit") + private AuditDTO audit; + + @Override + public String name() { + return name; + } + + @Override + public String comment() { + return comment; + } + + @Override + public Map properties() { + return properties; + } + + @Override + public int latestVersion() { + return latestVersion; + } + + @Override + public AuditDTO auditInfo() { + return audit; + } + + /** + * Creates a new builder for constructing a Model DTO. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for constructing a Model DTO. */ + public static class Builder { + private String name; + private String comment; + private Map properties; + private int latestVersion; + private AuditDTO audit; + + /** + * Sets the name of the model. + * + * @param name The name of the model. + * @return The builder. + */ + public Builder withName(String name) { + this.name = name; + return this; + } + + /** + * Sets the comment associated with the model. + * + * @param comment The comment associated with the model. + * @return The builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Sets the properties associated with the model. + * + * @param properties The properties associated with the model. + * @return The builder. + */ + public Builder withProperties(Map properties) { + this.properties = properties; + return this; + } + + /** + * Sets the latest version of the model. + * + * @param latestVersion The latest version of the model. + * @return The builder. + */ + public Builder withLatestVersion(int latestVersion) { + this.latestVersion = latestVersion; + return this; + } + + /** + * Sets the audit information associated with the model. + * + * @param audit The audit information associated with the model. + * @return The builder. + */ + public Builder withAudit(AuditDTO audit) { + this.audit = audit; + return this; + } + + /** + * Builds the model DTO. + * + * @return The model DTO. + */ + public ModelDTO build() { + Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be null or empty"); + Preconditions.checkArgument(latestVersion >= 0, "latestVersion cannot be negative"); + Preconditions.checkArgument(audit != null, "audit cannot be null"); + + return new ModelDTO(name, comment, properties, latestVersion, audit); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java b/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java new file mode 100644 index 00000000000..e887ba5bdb2 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Audit; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.model.ModelVersion; + +/** Represents a model version DTO (Data Transfer Object). */ +@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class ModelVersionDTO implements ModelVersion { + + @JsonProperty("version") + private int version; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("aliases") + private String[] aliases; + + @JsonProperty("uri") + private String uri; + + @JsonProperty("properties") + private Map properties; + + @JsonProperty("audit") + private AuditDTO audit; + + @Override + public Audit auditInfo() { + return audit; + } + + @Override + public int version() { + return version; + } + + @Override + public String comment() { + return comment; + } + + @Override + public String[] aliases() { + return aliases; + } + + @Override + public String uri() { + return uri; + } + + @Override + public Map properties() { + return properties; + } + + /** + * Creates a new builder for constructing a Model Version DTO. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for constructing a Model Version DTO. */ + public static class Builder { + private int version; + private String comment; + private String[] aliases; + private String uri; + private Map properties; + private AuditDTO audit; + + /** + * Sets the version number of the model version. + * + * @param version The version number. + * @return The builder. + */ + public Builder withVersion(int version) { + this.version = version; + return this; + } + + /** + * Sets the comment of the model version. + * + * @param comment The comment. + * @return The builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Sets the aliases of the model version. + * + * @param aliases The aliases. + * @return The builder. + */ + public Builder withAliases(String[] aliases) { + this.aliases = aliases; + return this; + } + + /** + * Sets the URI of the model version. + * + * @param uri The URI. + * @return The builder. + */ + public Builder withUri(String uri) { + this.uri = uri; + return this; + } + + /** + * Sets the properties of the model version. + * + * @param properties The properties. + * @return The builder. + */ + public Builder withProperties(Map properties) { + this.properties = properties; + return this; + } + + /** + * Sets the audit information of the model version. + * + * @param audit The audit information. + * @return The builder. + */ + public Builder withAudit(AuditDTO audit) { + this.audit = audit; + return this; + } + + /** + * Builds the Model Version DTO. + * + * @return The Model Version DTO. + */ + public ModelVersionDTO build() { + Preconditions.checkArgument(version >= 0, "Version must be non-negative"); + Preconditions.checkArgument(StringUtils.isNotBlank(uri), "URI cannot be null or empty"); + Preconditions.checkArgument(audit != null, "Audit cannot be null"); + + return new ModelVersionDTO(version, comment, aliases, uri, properties, audit); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java index 3da6579676d..d543ddb1649 100644 --- a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java +++ b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.dto.requests; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; import com.google.common.base.Preconditions; import java.util.Map; import javax.annotation.Nullable; @@ -54,11 +54,6 @@ public class CatalogCreateRequest implements RESTRequest { @JsonProperty("properties") private final Map properties; - /** Default constructor for CatalogCreateRequest. */ - public CatalogCreateRequest() { - this(null, null, null, null, null); - } - /** * Constructor for CatalogCreateRequest. * @@ -68,34 +63,24 @@ public CatalogCreateRequest() { * @param comment The comment for the catalog. * @param properties The properties for the catalog. */ + @JsonCreator public CatalogCreateRequest( - String name, - Catalog.Type type, - String provider, - String comment, - Map properties) { + @JsonProperty("name") String name, + @JsonProperty("type") Catalog.Type type, + @JsonProperty("provider") String provider, + @JsonProperty("comment") String comment, + @JsonProperty("properties") Map properties) { this.name = name; this.type = type; - this.provider = provider; this.comment = comment; this.properties = properties; - } - /** - * Sets the provider of the catalog if it is null. The value of provider in the request can be - * null if the catalog is a managed catalog. For such request, the value will be set when it is - * deserialized. - * - * @param provider The provider of the catalog. - */ - @JsonSetter(value = "provider") - public void setProvider(String provider) { if (StringUtils.isNotBlank(provider)) { this.provider = provider; } else if (type != null && type.supportsManagedCatalog()) { this.provider = CatalogProvider.shortNameForManagedCatalog(type); } else { - throw new IllegalStateException( + throw new IllegalArgumentException( "Provider cannot be null for catalog type " + type + " that doesn't support managed catalog"); diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java new file mode 100644 index 00000000000..b9cd1916174 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to register a model. */ +@Getter +@ToString +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class ModelRegisterRequest implements RESTRequest { + + @JsonProperty("name") + private String name; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(name), "\"name\" field is required and cannot be empty"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java new file mode 100644 index 00000000000..24e5932932c --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to link a model version. */ +@Getter +@ToString +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class ModelVersionLinkRequest implements RESTRequest { + + @JsonProperty("uri") + private String uri; + + @JsonProperty("aliases") + private String[] aliases; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(uri), "\"uri\" field is required and cannot be empty"); + + if (aliases != null && aliases.length > 0) { + for (String alias : aliases) { + Preconditions.checkArgument( + StringUtils.isNotBlank(alias), "alias must not be null or empty"); + Preconditions.checkArgument( + !NumberUtils.isCreatable(alias), "alias must not be a number or a number string"); + } + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java new file mode 100644 index 00000000000..ac51cdd647c --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.dto.model.ModelDTO; + +/** Response for model response. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelResponse extends BaseResponse { + + @JsonProperty("model") + private final ModelDTO model; + + /** + * Constructor for ModelResponse. + * + * @param model The model DTO object. + */ + public ModelResponse(ModelDTO model) { + super(0); + this.model = model; + } + + /** Default constructor for ModelResponse. (Used for Jackson deserialization.) */ + public ModelResponse() { + super(); + this.model = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(model != null, "model must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java new file mode 100644 index 00000000000..4d3551e1397 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** Represents a response for a list of model versions. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelVersionListResponse extends BaseResponse { + + @JsonProperty("versions") + private int[] versions; + + /** + * Constructor for ModelVersionListResponse. + * + * @param versions The list of model versions. + */ + public ModelVersionListResponse(int[] versions) { + super(0); + this.versions = versions; + } + + /** Default constructor for ModelVersionListResponse. (Used for Jackson deserialization.) */ + public ModelVersionListResponse() { + super(); + this.versions = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + Preconditions.checkArgument(versions != null, "versions cannot be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java new file mode 100644 index 00000000000..8b21472833b --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.dto.model.ModelVersionDTO; + +/** Represents a response for a model version. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelVersionResponse extends BaseResponse { + + @JsonProperty("modelVersion") + private final ModelVersionDTO modelVersion; + + /** + * Constructor for ModelVersionResponse. + * + * @param modelVersion The model version DTO object. + */ + public ModelVersionResponse(ModelVersionDTO modelVersion) { + super(0); + this.modelVersion = modelVersion; + } + + /** Default constructor for ModelVersionResponse. (Used for Jackson deserialization.) */ + public ModelVersionResponse() { + super(); + this.modelVersion = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(modelVersion != null, "modelVersion must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java index 254de8c3245..ce63398e605 100644 --- a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java @@ -51,6 +51,8 @@ import org.apache.gravitino.dto.credential.CredentialDTO; import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.messaging.TopicDTO; +import org.apache.gravitino.dto.model.ModelDTO; +import org.apache.gravitino.dto.model.ModelVersionDTO; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.DistributionDTO; import org.apache.gravitino.dto.rel.SortOrderDTO; @@ -80,6 +82,8 @@ import org.apache.gravitino.dto.tag.TagDTO; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.messaging.Topic; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.expressions.Expression; @@ -629,6 +633,39 @@ public static TopicDTO toDTO(Topic topic) { .build(); } + /** + * Converts a Model to a ModelDTO. + * + * @param model The model to be converted. + * @return The model DTO. + */ + public static ModelDTO toDTO(Model model) { + return ModelDTO.builder() + .withName(model.name()) + .withComment(model.comment()) + .withProperties(model.properties()) + .withLatestVersion(model.latestVersion()) + .withAudit(toDTO(model.auditInfo())) + .build(); + } + + /** + * Converts a ModelVersion to a ModelVersionDTO. + * + * @param modelVersion The model version to be converted. + * @return The model version DTO. + */ + public static ModelVersionDTO toDTO(ModelVersion modelVersion) { + return ModelVersionDTO.builder() + .withVersion(modelVersion.version()) + .withComment(modelVersion.comment()) + .withAliases(modelVersion.aliases()) + .withUri(modelVersion.uri()) + .withProperties(modelVersion.properties()) + .withAudit(toDTO(modelVersion.auditInfo())) + .build(); + } + /** * Converts an array of Columns to an array of ColumnDTOs. * diff --git a/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java b/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java new file mode 100644 index 00000000000..39e4628eca2 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.Map; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelDTO { + + @Test + public void testModelSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + Map props = ImmutableMap.of("key", "value"); + + ModelDTO modelDTO = + ModelDTO.builder() + .withName("model_test") + .withComment("model comment") + .withLatestVersion(0) + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson = JsonUtils.objectMapper().writeValueAsString(modelDTO); + ModelDTO deserModelDTO = JsonUtils.objectMapper().readValue(serJson, ModelDTO.class); + Assertions.assertEquals(modelDTO, deserModelDTO); + + // Test with null comment and properties + ModelDTO modelDTO1 = + ModelDTO.builder().withName("model_test").withLatestVersion(0).withAudit(audit).build(); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(modelDTO1); + ModelDTO deserModelDTO1 = JsonUtils.objectMapper().readValue(serJson1, ModelDTO.class); + Assertions.assertEquals(modelDTO1, deserModelDTO1); + Assertions.assertNull(deserModelDTO1.comment()); + Assertions.assertNull(deserModelDTO1.properties()); + } + + @Test + public void testInvalidModelDTO() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().withName("model_test").withLatestVersion(-1).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().withName("model_test").withLatestVersion(0).build(); + }); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java b/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java new file mode 100644 index 00000000000..5251246c377 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.Map; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelVersionDTO { + + @Test + public void testModelVersionSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + Map props = ImmutableMap.of("key", "value"); + + ModelVersionDTO modelVersionDTO = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {"alias1", "alias2"}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO); + ModelVersionDTO deserModelVersionDTO = + JsonUtils.objectMapper().readValue(serJson, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO, deserModelVersionDTO); + + // Test with null aliases + ModelVersionDTO modelVersionDTO1 = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO1); + ModelVersionDTO deserModelVersionDTO1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO1, deserModelVersionDTO1); + Assertions.assertNull(deserModelVersionDTO1.aliases()); + + // Test with empty aliases + ModelVersionDTO modelVersionDTO2 = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO2); + ModelVersionDTO deserModelVersionDTO2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO2, deserModelVersionDTO2); + Assertions.assertArrayEquals(new String[] {}, deserModelVersionDTO2.aliases()); + + // Test with null comment and properties + ModelVersionDTO modelVersionDTO3 = + ModelVersionDTO.builder().withVersion(0).withUri("uri").withAudit(audit).build(); + + String serJson3 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO3); + ModelVersionDTO deserModelVersionDTO3 = + JsonUtils.objectMapper().readValue(serJson3, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO3, deserModelVersionDTO3); + Assertions.assertNull(deserModelVersionDTO3.comment()); + Assertions.assertNull(deserModelVersionDTO3.properties()); + } + + @Test + public void testInvalidModelVersionDTO() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(-1).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).withUri("").build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).withUri("uri").build(); + }); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java index b4b7383a7ea..3b5221ff1a7 100644 --- a/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java @@ -57,19 +57,24 @@ public void testCatalogCreateRequestSerDe() throws JsonProcessingException { String serJson1 = JsonUtils.objectMapper().writeValueAsString(request1); CatalogCreateRequest deserRequest1 = JsonUtils.objectMapper().readValue(serJson1, CatalogCreateRequest.class); - Assertions.assertEquals( deserRequest1.getType().name().toLowerCase(Locale.ROOT), deserRequest1.getProvider()); Assertions.assertNull(deserRequest1.getComment()); Assertions.assertNull(deserRequest1.getProperties()); + String json = "{\"name\":\"catalog_test\",\"type\":\"model\"}"; + CatalogCreateRequest deserRequest2 = + JsonUtils.objectMapper().readValue(json, CatalogCreateRequest.class); + Assertions.assertEquals("model", deserRequest2.getProvider()); + // Test using null provider with catalog type doesn't support managed catalog - CatalogCreateRequest request2 = - new CatalogCreateRequest("catalog_test", Catalog.Type.RELATIONAL, null, null, null); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> new CatalogCreateRequest("catalog_test", Catalog.Type.RELATIONAL, null, null, null)); - String serJson2 = JsonUtils.objectMapper().writeValueAsString(request2); + String json1 = "{\"name\":\"catalog_test\",\"type\":\"relational\"}"; Assertions.assertThrows( JsonMappingException.class, - () -> JsonUtils.objectMapper().readValue(serJson2, CatalogCreateRequest.class)); + () -> JsonUtils.objectMapper().readValue(json1, CatalogCreateRequest.class)); } } diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java new file mode 100644 index 00000000000..09dbdb8c312 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelRegisterRequest { + + @Test + public void testModelRegisterRequestSerDe() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + ModelRegisterRequest req = new ModelRegisterRequest("model", "comment", props); + + String serJson = JsonUtils.objectMapper().writeValueAsString(req); + ModelRegisterRequest deserReq = + JsonUtils.objectMapper().readValue(serJson, ModelRegisterRequest.class); + Assertions.assertEquals(req, deserReq); + + // Test with null comment and properties + ModelRegisterRequest req1 = new ModelRegisterRequest("model", null, null); + String serJson1 = JsonUtils.objectMapper().writeValueAsString(req1); + ModelRegisterRequest deserReq1 = + JsonUtils.objectMapper().readValue(serJson1, ModelRegisterRequest.class); + Assertions.assertEquals(req1, deserReq1); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java new file mode 100644 index 00000000000..4c0df6d73e8 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelVersionLinkRequest { + + @Test + public void testModelVersionLinkRequestSerDe() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + ModelVersionLinkRequest request = + new ModelVersionLinkRequest("uri", new String[] {"alias1", "alias2"}, "comment", props); + + String serJson = JsonUtils.objectMapper().writeValueAsString(request); + ModelVersionLinkRequest deserRequest = + JsonUtils.objectMapper().readValue(serJson, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request, deserRequest); + + // Test with null aliases + ModelVersionLinkRequest request1 = new ModelVersionLinkRequest("uri", null, "comment", props); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(request1); + ModelVersionLinkRequest deserRequest1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request1, deserRequest1); + Assertions.assertNull(deserRequest1.getAliases()); + + // Test with empty aliases + ModelVersionLinkRequest request2 = + new ModelVersionLinkRequest("uri", new String[] {}, "comment", props); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(request2); + ModelVersionLinkRequest deserRequest2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request2, deserRequest2); + Assertions.assertEquals(0, deserRequest2.getAliases().length); + + // Test with null comment and properties + ModelVersionLinkRequest request3 = + new ModelVersionLinkRequest("uri", new String[] {"alias1", "alias2"}, null, null); + + String serJson3 = JsonUtils.objectMapper().writeValueAsString(request3); + ModelVersionLinkRequest deserRequest3 = + JsonUtils.objectMapper().readValue(serJson3, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request3, deserRequest3); + Assertions.assertNull(deserRequest3.getComment()); + Assertions.assertNull(deserRequest3.getProperties()); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java index 5f947820222..57813c0bc64 100644 --- a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java @@ -26,8 +26,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.time.Instant; +import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.authorization.Privileges; @@ -41,6 +43,8 @@ import org.apache.gravitino.dto.authorization.RoleDTO; import org.apache.gravitino.dto.authorization.SecurableObjectDTO; import org.apache.gravitino.dto.authorization.UserDTO; +import org.apache.gravitino.dto.model.ModelDTO; +import org.apache.gravitino.dto.model.ModelVersionDTO; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.TableDTO; import org.apache.gravitino.dto.rel.partitioning.Partitioning; @@ -390,4 +394,76 @@ void testFileLocationResponseException() { FileLocationResponse response = new FileLocationResponse(); assertThrows(IllegalArgumentException.class, () -> response.validate()); } + + @Test + void testModelResponse() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + ModelDTO modelDTO = + ModelDTO.builder() + .withName("model1") + .withLatestVersion(0) + .withComment("comment1") + .withProperties(props) + .withAudit(audit) + .build(); + + ModelResponse response = new ModelResponse(modelDTO); + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + ModelResponse deserResponse = JsonUtils.objectMapper().readValue(serJson, ModelResponse.class); + + assertEquals(response, deserResponse); + + ModelResponse response1 = new ModelResponse(); + assertThrows(IllegalArgumentException.class, response1::validate); + } + + @Test + void testModelVersionListResponse() throws JsonProcessingException { + ModelVersionListResponse response1 = new ModelVersionListResponse(new int[] {}); + assertDoesNotThrow(response1::validate); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(response1); + ModelVersionListResponse deserResponse1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionListResponse.class); + assertEquals(response1, deserResponse1); + assertArrayEquals(new int[] {}, deserResponse1.getVersions()); + + ModelVersionListResponse response2 = new ModelVersionListResponse(new int[] {1, 2}); + assertDoesNotThrow(response2::validate); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(response2); + ModelVersionListResponse deserResponse2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionListResponse.class); + assertEquals(response2, deserResponse2); + assertArrayEquals(new int[] {1, 2}, deserResponse2.getVersions()); + } + + @Test + void testModelVersionResponse() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + ModelVersionDTO modelVersionDTO = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {"alias1", "alias2"}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + ModelVersionResponse response = new ModelVersionResponse(modelVersionDTO); + response.validate(); // No exception thrown + + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + ModelVersionResponse deserResponse = + JsonUtils.objectMapper().readValue(serJson, ModelVersionResponse.class); + assertEquals(response, deserResponse); + + ModelVersionResponse response1 = new ModelVersionResponse(); + assertThrows(IllegalArgumentException.class, response1::validate); + } } diff --git a/docs/open-api/models.yaml b/docs/open-api/models.yaml new file mode 100644 index 00000000000..713a7037cd6 --- /dev/null +++ b/docs/open-api/models.yaml @@ -0,0 +1,561 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- + +paths: + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + + get: + tags: + - model + summary: List models + operationId: listModels + responses: + "200": + $ref: "./openapi.yaml#/components/responses/EntityListResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + post: + tags: + - model + summary: Register model + operationId: registerModel + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ModelRegisterRequest" + examples: + ModelRegisterRequest: + $ref: "#/components/examples/ModelRegisterRequest" + responses: + "200": + $ref: "#/components/responses/ModelResponse" + "409": + description: Conflict - The target model already exists + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + ModelAlreadyExistsErrorResponse: + $ref: "#/components/examples/ModelAlreadyExistsException" + "404": + description: Not Found - The schema does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + + get: + tags: + - model + summary: Get model + operationId: getModel + description: Returns the specified model object + responses: + "200": + $ref: "#/components/responses/ModelResponse" + "404": + description: Not Found - The target fileset does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchCatalogException: + $ref: "./catalogs.yaml#/components/examples/NoSuchCatalogException" + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model + operationId: deleteModel + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + post: + tags: + - model + summary: link model version + operationId: linkModelVersion + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ModelVersionLinkRequest" + examples: + ModelVersionLinkRequest: + $ref: "#/components/examples/ModelVersionLinkRequest" + responses: + "200": + $ref: "./openapi.yaml#/components/responses/BaseResponse" + "404": + description: Not Found - The target model does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "409": + description: Conflict - The model version aliases already exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + ModelVersionAliasesAlreadyExistException: + $ref: "#/components/examples/ModelVersionAliasesAlreadyExistException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + + get: + tags: + - model + summary: List model versions + operationId: listModelVersions + responses: + "200": + $ref: "#/components/responses/ModelVersionListResponse" + "404": + description: Not Found - The target model does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions/{version}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + - $ref: "#/components/parameters/version" + + get: + tags: + - model + summary: Get model version + operationId: getModelVersion + responses: + "200": + $ref: "#/components/responses/ModelVersionResponse" + "404": + description: Not Found - The target model version does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelVersionException: + $ref: "#/components/examples/NoSuchModelVersionException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model version + operationId: deleteModelVersion + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/aliases/{alias}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + - $ref: "#/components/parameters/alias" + + get: + tags: + - model + summary: Get model version by alias + operationId: getModelVersionByAlias + responses: + "200": + $ref: "#/components/responses/ModelVersionResponse" + "404": + description: Not Found - The target model version does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelVersionException: + $ref: "#/components/examples/NoSuchModelVersionException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model version by alias + operationId: deleteModelVersionByAlias + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + +components: + parameters: + version: + name: version + in: path + required: true + description: The version of the model + schema: + type: integer + alias: + name: alias + in: path + required: true + description: The alias of the model version + schema: + type: string + + schemas: + Model: + type: object + required: + - name + - audit + - latestVersion + properties: + name: + type: string + description: The name of the model + latestVersion: + type: integer + description: The latest version of the model + comment: + type: string + description: The comment of the fileset + nullable: true + properties: + type: object + description: The properties of the fileset + nullable: true + default: {} + additionalProperties: + type: string + audit: + $ref: "./openapi.yaml#/components/schemas/Audit" + + ModelVersion: + type: object + required: + - uri + - version + - audit + properties: + uri: + type: string + description: The uri of the model version + version: + type: integer + description: The version of the model + aliases: + type: array + description: The aliases of the model version + nullable: true + items: + type: string + comment: + type: string + description: The comment of the model version + nullable: true + properties: + type: object + description: The properties of the model version + nullable: true + default: {} + additionalProperties: + type: string + audit: + $ref: "./openapi.yaml#/components/schemas/Audit" + + ModelRegisterRequest: + type: object + required: + - name + properties: + name: + type: string + description: The name of the model. Can not be empty. + comment: + type: string + description: The comment of the model. Can be empty. + nullable: true + properties: + type: object + description: The properties of the model. Can be empty. + nullable: true + default: {} + additionalProperties: + type: string + + ModelVersionLinkRequest: + type: object + required: + - uri + properties: + uri: + type: string + description: The uri of the model version + aliases: + type: array + description: The aliases of the model version + nullable: true + items: + type: string + comment: + type: string + description: The comment of the model version + nullable: true + properties: + type: object + description: The properties of the model version + nullable: true + default: {} + additionalProperties: + type: string + + responses: + ModelResponse: + description: The response of model object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + model: + $ref: "#/components/schemas/Model" + examples: + ModelResponse: + $ref: "#/components/examples/ModelResponse" + ModelVersionListResponse: + description: The response of model version list + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + versions: + type: array + description: The list of model versions + items: + format: int32 + examples: + ModelVersionListResponse: + $ref: "#/components/examples/ModelVersionListResponse" + ModelVersionResponse: + description: The response of model version object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + modelVersion: + $ref: "#/components/schemas/ModelVersion" + examples: + ModelResponse: + $ref: "#/components/examples/ModelVersionResponse" + + examples: + ModelRegisterRequest: + value: { + "name": "model1", + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + } + } + + ModelVersionLinkRequest: + value: { + "uri": "hdfs://path/to/model", + "aliases": ["alias1", "alias2"], + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + } + } + + ModelResponse: + value: { + "code": 0, + "model" : { + "name": "model1", + "latestVersion": 0, + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + }, + "audit": { + "creator": "user1", + "createTime": "2021-01-01T00:00:00Z", + "lastModifier": "user1", + "lastModifiedTime": "2021-01-01T00:00:00Z" + } + } + } + + ModelVersionListResponse: + value: { + "code": 0, + "versions": [0, 1, 2] + } + + ModelVersionResponse: + value: { + "code": 0, + "modelVersion" : { + "uri": "hdfs://path/to/model", + "version": 0, + "aliases": ["alias1", "alias2"], + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + }, + "audit": { + "creator": "user1", + "createTime": "2021-01-01T00:00:00Z", + "lastModifier": "user1", + "lastModifiedTime": "2021-01-01T00:00:00Z" + } + } + } + + ModelAlreadyExistsException: + value: { + "code": 1004, + "type": "ModelAlreadyExistsException", + "message": "Model already exists", + "stack": [ + "org.apache.gravitino.exceptions.ModelAlreadyExistsException: Model already exists" + ] + } + + NoSuchModelException: + value: { + "code": 1003, + "type": "NoSuchModelException", + "message": "Model does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchModelException: Model does not exist" + ] + } + + ModelVersionAliasesAlreadyExistException: + value: { + "code": 1004, + "type": "ModelVersionAliasesAlreadyExistException", + "message": "Model version aliases already exist", + "stack": [ + "org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException: Model version aliases already exist" + ] + } + + NoSuchModelVersionException: + value: { + "code": 1003, + "type": "NoSuchModelVersionException", + "message": "Model version does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchModelVersionException: Model version does not exist" + ] + } diff --git a/docs/open-api/openapi.yaml b/docs/open-api/openapi.yaml index dd0564a7f9c..d0c941ab471 100644 --- a/docs/open-api/openapi.yaml +++ b/docs/open-api/openapi.yaml @@ -113,6 +113,20 @@ paths: /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/topics/{topic}: $ref: "./topics.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1topics~1%7Btopic%7D" + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1versions" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions/{version}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1versions~1%7Bversion%7D" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/aliases/{alias}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1aliases~1%7Balias%7D" /metalakes/{metalake}/users: $ref: "./users.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1users" @@ -430,6 +444,14 @@ components: schema: type: string + model: + name: model + in: path + description: The name of the model + required: true + schema: + type: string + tag: name: tag in: path @@ -476,6 +498,7 @@ components: - "COLUMN" - "FILESET" - "TOPIC" + - "MODEL" - "ROLE" metadataObjectFullName: name: metadataObjectFullName diff --git a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java index 63e53aefd59..2afc65482b3 100644 --- a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java @@ -27,6 +27,7 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.catalog.ModelDispatcher; import org.apache.gravitino.catalog.PartitionDispatcher; import org.apache.gravitino.catalog.SchemaDispatcher; import org.apache.gravitino.catalog.TableDispatcher; @@ -118,6 +119,7 @@ protected void configure() { bind(gravitinoEnv.credentialOperationDispatcher()) .to(CredentialOperationDispatcher.class) .ranked(1); + bind(gravitinoEnv.modelDispatcher()).to(ModelDispatcher.class).ranked(1); } }); register(JsonProcessingExceptionMapper.class); diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java index faf94f50648..b71219b0453 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java @@ -32,6 +32,8 @@ import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.MetalakeNotInUseException; +import org.apache.gravitino.exceptions.ModelAlreadyExistsException; +import org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NonEmptyCatalogException; import org.apache.gravitino.exceptions.NonEmptyMetalakeException; @@ -126,6 +128,11 @@ public static Response handleCredentialException( return CredentialExceptionHandler.INSTANCE.handle(op, metadataObjectName, "", e); } + public static Response handleModelException( + OperationType op, String model, String schema, Exception e) { + return ModelExceptionHandler.INSTANCE.handle(op, model, schema, e); + } + public static Response handleTestConnectionException(Exception e) { ErrorResponse response; if (e instanceof IllegalArgumentException) { @@ -729,6 +736,44 @@ public Response handle(OperationType op, String name, String parent, Exception e } } + private static class ModelExceptionHandler extends BaseExceptionHandler { + private static final ExceptionHandler INSTANCE = new ModelExceptionHandler(); + + private static String getModelErrorMsg( + String model, String operation, String schema, String reason) { + return String.format( + "Failed to operate model(s)%s operation [%s] under schema [%s], reason [%s]", + model, operation, schema, reason); + } + + @Override + public Response handle(OperationType op, String model, String schema, Exception e) { + String formatted = StringUtil.isBlank(model) ? "" : " [" + model + "]"; + String errorMsg = getModelErrorMsg(formatted, op.name(), schema, getErrorMsg(e)); + LOG.warn(errorMsg, e); + + if (e instanceof IllegalArgumentException) { + return Utils.illegalArguments(errorMsg, e); + + } else if (e instanceof NotFoundException) { + return Utils.notFound(errorMsg, e); + + } else if (e instanceof ModelAlreadyExistsException + || e instanceof ModelVersionAliasesAlreadyExistException) { + return Utils.alreadyExists(errorMsg, e); + + } else if (e instanceof ForbiddenException) { + return Utils.forbidden(errorMsg, e); + + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + + } else { + return super.handle(op, model, schema, e); + } + } + } + @VisibleForTesting static class BaseExceptionHandler extends ExceptionHandler { diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java new file mode 100644 index 00000000000..fd507821086 --- /dev/null +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.ModelDispatcher; +import org.apache.gravitino.dto.requests.ModelRegisterRequest; +import org.apache.gravitino.dto.requests.ModelVersionLinkRequest; +import org.apache.gravitino.dto.responses.BaseResponse; +import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.ModelResponse; +import org.apache.gravitino.dto.responses.ModelVersionListResponse; +import org.apache.gravitino.dto.responses.ModelVersionResponse; +import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.metrics.MetricNames; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; +import org.apache.gravitino.server.web.Utils; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Path("metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models") +public class ModelOperations { + + private static final Logger LOG = LoggerFactory.getLogger(ModelOperations.class); + + private final ModelDispatcher modelDispatcher; + + @Context private HttpServletRequest httpRequest; + + @Inject + public ModelOperations(ModelDispatcher modelDispatcher) { + this.modelDispatcher = modelDispatcher; + } + + @GET + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-model", absolute = true) + public Response listModels( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema) { + LOG.info("Received list models request for schema: {}.{}.{}", metalake, catalog, schema); + Namespace modelNs = NamespaceUtil.ofModel(metalake, catalog, schema); + + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier[] modelIds = modelDispatcher.listModels(modelNs); + modelIds = modelIds == null ? new NameIdentifier[0] : modelIds; + LOG.info("List {} models under schema {}", modelIds.length, modelNs); + return Utils.ok(new EntityListResponse(modelIds)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LIST, "", schema, e); + } + } + + @GET + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model", absolute = true) + public Response getModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received get model request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + Model m = modelDispatcher.getModel(modelId); + LOG.info("Model got: {}", modelId); + return Utils.ok(new ModelResponse(DTOConverters.toDTO(m))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.GET, model, schema, e); + } + } + + @POST + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "register-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "register-model", absolute = true) + public Response registerModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + ModelRegisterRequest request) { + LOG.info( + "Received register model request: {}.{}.{}.{}", + metalake, + catalog, + schema, + request.getName()); + + try { + request.validate(); + NameIdentifier modelId = + NameIdentifierUtil.ofModel(metalake, catalog, schema, request.getName()); + + return Utils.doAs( + httpRequest, + () -> { + Model m = + modelDispatcher.registerModel( + modelId, request.getComment(), request.getProperties()); + LOG.info("Model registered: {}", modelId); + return Utils.ok(new ModelResponse(DTOConverters.toDTO(m))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.REGISTER, request.getName(), schema, e); + } + } + + @DELETE + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model", absolute = true) + public Response deleteModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received delete model request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModel(modelId); + if (!deleted) { + LOG.warn("Cannot find to be deleted model {} under schema {}", model, schema); + } else { + LOG.info("Model deleted: {}", modelId); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.DELETE, model, schema, e); + } + } + + @GET + @Path("{model}/versions") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-model-versions." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-model-versions", absolute = true) + public Response listModelVersions( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received list model versions request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + int[] versions = modelDispatcher.listModelVersions(modelId); + versions = versions == null ? new int[0] : versions; + LOG.info("List {} versions of model {}", versions.length, modelId); + return Utils.ok(new ModelVersionListResponse(versions)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LIST_VERSIONS, model, schema, e); + } + } + + @GET + @Path("{model}/versions/{version}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model-version", absolute = true) + public Response getModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("version") int version) { + LOG.info( + "Received get model version request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + version); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + ModelVersion mv = modelDispatcher.getModelVersion(modelId, version); + LOG.info("Model version got: {}.{}", modelId, version); + return Utils.ok(new ModelVersionResponse(DTOConverters.toDTO(mv))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.GET, versionString(model, version), schema, e); + } + } + + @GET + @Path("{model}/aliases/{alias}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model-alias." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model-alias", absolute = true) + public Response getModelVersionByAlias( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("alias") String alias) { + LOG.info( + "Received get model version alias request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + alias); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + ModelVersion mv = modelDispatcher.getModelVersion(modelId, alias); + LOG.info("Model version alias got: {}.{}", modelId, alias); + return Utils.ok(new ModelVersionResponse(DTOConverters.toDTO(mv))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.GET, aliasString(model, alias), schema, e); + } + } + + @POST + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "link-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "link-model-version", absolute = true) + public Response linkModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + ModelVersionLinkRequest request) { + LOG.info("Received link model version request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + request.validate(); + + return Utils.doAs( + httpRequest, + () -> { + modelDispatcher.linkModelVersion( + modelId, + request.getUri(), + request.getAliases(), + request.getComment(), + request.getProperties()); + LOG.info("Model version linked: {}", modelId); + return Utils.ok(new BaseResponse()); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LINK, model, schema, e); + } + } + + @DELETE + @Path("{model}/versions/{version}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model-version", absolute = true) + public Response deleteModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("version") int version) { + LOG.info( + "Received delete model version request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + version); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModelVersion(modelId, version); + if (!deleted) { + LOG.warn("Cannot find to be deleted version {} in model {}", version, model); + } else { + LOG.info("Model version deleted: {}.{}", modelId, version); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.DELETE, versionString(model, version), schema, e); + } + } + + @DELETE + @Path("{model}/aliases/{alias}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model-alias." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model-alias", absolute = true) + public Response deleteModelVersionByAlias( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("alias") String alias) { + LOG.info( + "Received delete model version by alias request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + alias); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModelVersion(modelId, alias); + if (!deleted) { + LOG.warn( + "Cannot find to be deleted model version by alias {} in model {}", alias, model); + } else { + LOG.info("Model version by alias deleted: {}.{}", modelId, alias); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.DELETE, aliasString(model, alias), schema, e); + } + } + + private String versionString(String model, int version) { + return model + " version(" + version + ")"; + } + + private String aliasString(String model, String alias) { + return model + " alias(" + alias + ")"; + } +} diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java b/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java index 8d4bc322ae7..2b8abd91f1d 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java @@ -35,4 +35,7 @@ public enum OperationType { REVOKE, ASSOCIATE, SET, + REGISTER, // An operation to register a model + LIST_VERSIONS, // An operation to list versions of a model + LINK // An operation to link a version to a model } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java new file mode 100644 index 00000000000..42e48d0302f --- /dev/null +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java @@ -0,0 +1,843 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web.rest; + +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.time.Instant; +import java.util.Map; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.ModelDispatcher; +import org.apache.gravitino.dto.requests.ModelRegisterRequest; +import org.apache.gravitino.dto.requests.ModelVersionLinkRequest; +import org.apache.gravitino.dto.responses.BaseResponse; +import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.ErrorConstants; +import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.dto.responses.ModelResponse; +import org.apache.gravitino.dto.responses.ModelVersionListResponse; +import org.apache.gravitino.dto.responses.ModelVersionResponse; +import org.apache.gravitino.exceptions.ModelAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchModelException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; +import org.apache.gravitino.rest.RESTUtils; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.glassfish.jersey.internal.inject.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.JerseyTest; +import org.glassfish.jersey.test.TestProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelOperations extends JerseyTest { + + private static class MockServletRequestFactory extends ServletRequestFactoryBase { + + @Override + public HttpServletRequest get() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRemoteUser()).thenReturn(null); + return request; + } + } + + private ModelDispatcher modelDispatcher = mock(ModelDispatcher.class); + + private AuditInfo testAuditInfo = + AuditInfo.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + private Map properties = ImmutableMap.of("key1", "value"); + + private String metalake = "metalake_for_model_test"; + + private String catalog = "catalog_for_model_test"; + + private String schema = "schema_for_model_test"; + + private Namespace modelNs = NamespaceUtil.ofModel(metalake, catalog, schema); + + @Override + protected Application configure() { + try { + forceSet( + TestProperties.CONTAINER_PORT, String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(ModelOperations.class); + resourceConfig.register( + new AbstractBinder() { + @Override + protected void configure() { + bind(modelDispatcher).to(ModelDispatcher.class).ranked(2); + bindFactory(TestModelOperations.MockServletRequestFactory.class) + .to(HttpServletRequest.class); + } + }); + + return resourceConfig; + } + + @Test + public void testListModels() { + NameIdentifier modelId1 = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + NameIdentifier modelId2 = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model2"); + NameIdentifier[] modelIds = new NameIdentifier[] {modelId1, modelId2}; + when(modelDispatcher.listModels(modelNs)).thenReturn(modelIds); + + Response response = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + + EntityListResponse resp = response.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp.getCode()); + Assertions.assertArrayEquals(modelIds, resp.identifiers()); + + // Test mock return null for listModels + when(modelDispatcher.listModels(modelNs)).thenReturn(null); + Response resp1 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + EntityListResponse resp2 = resp1.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp2.getCode()); + Assertions.assertEquals(0, resp2.identifiers().length); + + // Test mock return empty array for listModels + when(modelDispatcher.listModels(modelNs)).thenReturn(new NameIdentifier[0]); + Response resp3 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + EntityListResponse resp4 = resp3.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp4.getCode()); + Assertions.assertEquals(0, resp4.identifiers().length); + + // Test mock throw NoSuchSchemaException + doThrow(new NoSuchSchemaException("mock error")).when(modelDispatcher).listModels(modelNs); + Response resp5 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchSchemaException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).listModels(modelNs); + Response resp6 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp6.getStatus()); + + ErrorResponse errorResp1 = resp6.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetModel() { + Model mockModel = mockModel("model1", "comment1", 0); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.getModel(modelId)).thenReturn(mockModel); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelResponse modelResp = resp.readEntity(ModelResponse.class); + Assertions.assertEquals(0, modelResp.getCode()); + + Model resultModel = modelResp.getModel(); + compare(mockModel, resultModel); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")).when(modelDispatcher).getModel(modelId); + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).getModel(modelId); + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testRegisterModel() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + Model mockModel = mockModel("model1", "comment1", 0); + when(modelDispatcher.registerModel(modelId, "comment1", properties)).thenReturn(mockModel); + + ModelRegisterRequest req = new ModelRegisterRequest("model1", "comment1", properties); + Response resp = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelResponse modelResp = resp.readEntity(ModelResponse.class); + Assertions.assertEquals(0, modelResp.getCode()); + compare(mockModel, modelResp.getModel()); + + // Test mock throw NoSuchSchemaException + doThrow(new NoSuchSchemaException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp1 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchSchemaException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw ModelAlreadyExistsException + doThrow(new ModelAlreadyExistsException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp2 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResp1.getCode()); + Assertions.assertEquals( + ModelAlreadyExistsException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp3 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + @Test + public void testDeleteModel() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.deleteModel(modelId)).thenReturn(true); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + DropResponse dropResp = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp.getCode()); + Assertions.assertTrue(dropResp.dropped()); + + // Test mock return false for deleteModel + when(modelDispatcher.deleteModel(modelId)).thenReturn(false); + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + DropResponse dropResp1 = resp1.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp1.getCode()); + Assertions.assertFalse(dropResp1.dropped()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).deleteModel(modelId); + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testListModelVersions() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + int[] versions = new int[] {0, 1, 2}; + when(modelDispatcher.listModelVersions(modelId)).thenReturn(versions); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelVersionListResponse versionListResp = resp.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp.getCode()); + Assertions.assertArrayEquals(versions, versionListResp.getVersions()); + + // Test mock return null for listModelVersions + when(modelDispatcher.listModelVersions(modelId)).thenReturn(null); + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + ModelVersionListResponse versionListResp1 = resp1.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp1.getCode()); + Assertions.assertEquals(0, versionListResp1.getVersions().length); + + // Test mock return empty array for listModelVersions + when(modelDispatcher.listModelVersions(modelId)).thenReturn(new int[0]); + Response resp2 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp2.getMediaType()); + + ModelVersionListResponse versionListResp2 = resp2.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp2.getCode()); + Assertions.assertEquals(0, versionListResp2.getVersions().length); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .listModelVersions(modelId); + Response resp3 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).listModelVersions(modelId); + Response resp4 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp1 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + ModelVersion mockModelVersion = + mockModelVersion(0, "uri1", new String[] {"alias1"}, "comment1"); + when(modelDispatcher.getModelVersion(modelIdent, 0)).thenReturn(mockModelVersion); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelVersionResponse versionResp = resp.readEntity(ModelVersionResponse.class); + Assertions.assertEquals(0, versionResp.getCode()); + compare(mockModelVersion, versionResp.getModelVersion()); + + // Test mock throw NoSuchModelVersionException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, 0); + + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, 0); + + Response resp2 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + + // Test get model version by alias + when(modelDispatcher.getModelVersion(modelIdent, "alias1")).thenReturn(mockModelVersion); + + Response resp3 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + ModelVersionResponse versionResp1 = resp3.readEntity(ModelVersionResponse.class); + Assertions.assertEquals(0, versionResp1.getCode()); + compare(mockModelVersion, versionResp1.getModelVersion()); + + // Test mock throw NoSuchModelVersionException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, "alias1"); + + Response resp4 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp2 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp2.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp2.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, "alias1"); + + Response resp5 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp3 = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp3.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp3.getType()); + } + + @Test + public void testLinkModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + doNothing() + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + ModelVersionLinkRequest req = + new ModelVersionLinkRequest("uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + BaseResponse baseResponse = resp.readEntity(BaseResponse.class); + Assertions.assertEquals(0, baseResponse.getCode()); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw ModelVersionAliasesAlreadyExistException + doThrow(new ModelAlreadyExistsException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResp1.getCode()); + Assertions.assertEquals( + ModelAlreadyExistsException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp3 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + @Test + public void testDeleteModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.deleteModelVersion(modelIdent, 0)).thenReturn(true); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + DropResponse dropResp = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp.getCode()); + Assertions.assertTrue(dropResp.dropped()); + + // Test mock return false for deleteModelVersion + when(modelDispatcher.deleteModelVersion(modelIdent, 0)).thenReturn(false); + + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + DropResponse dropResp1 = resp1.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp1.getCode()); + Assertions.assertFalse(dropResp1.dropped()); + + // Test mock return true for deleteModelVersion using alias + when(modelDispatcher.deleteModelVersion(modelIdent, "alias1")).thenReturn(true); + + Response resp2 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp2.getMediaType()); + + DropResponse dropResp2 = resp2.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp2.getCode()); + + // Test mock return false for deleteModelVersion using alias + when(modelDispatcher.deleteModelVersion(modelIdent, "alias1")).thenReturn(false); + + Response resp3 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + DropResponse dropResp3 = resp3.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp3.getCode()); + Assertions.assertFalse(dropResp3.dropped()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .deleteModelVersion(modelIdent, 0); + + Response resp4 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp1 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException using alias + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .deleteModelVersion(modelIdent, "alias1"); + + Response resp5 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp2 = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + private String modelPath() { + return "/metalakes/" + metalake + "/catalogs/" + catalog + "/schemas/" + schema + "/models"; + } + + private Model mockModel(String modelName, String comment, int latestVersion) { + Model mockModel = mock(Model.class); + when(mockModel.name()).thenReturn(modelName); + when(mockModel.comment()).thenReturn(comment); + when(mockModel.latestVersion()).thenReturn(latestVersion); + when(mockModel.properties()).thenReturn(properties); + when(mockModel.auditInfo()).thenReturn(testAuditInfo); + return mockModel; + } + + private ModelVersion mockModelVersion(int version, String uri, String[] aliases, String comment) { + ModelVersion mockModelVersion = mock(ModelVersion.class); + when(mockModelVersion.version()).thenReturn(version); + when(mockModelVersion.uri()).thenReturn(uri); + when(mockModelVersion.aliases()).thenReturn(aliases); + when(mockModelVersion.comment()).thenReturn(comment); + when(mockModelVersion.properties()).thenReturn(properties); + when(mockModelVersion.auditInfo()).thenReturn(testAuditInfo); + return mockModelVersion; + } + + private void compare(Model left, Model right) { + Assertions.assertEquals(left.name(), right.name()); + Assertions.assertEquals(left.comment(), right.comment()); + Assertions.assertEquals(left.properties(), right.properties()); + + Assertions.assertNotNull(right.auditInfo()); + Assertions.assertEquals(left.auditInfo().creator(), right.auditInfo().creator()); + Assertions.assertEquals(left.auditInfo().createTime(), right.auditInfo().createTime()); + Assertions.assertEquals(left.auditInfo().lastModifier(), right.auditInfo().lastModifier()); + Assertions.assertEquals( + left.auditInfo().lastModifiedTime(), right.auditInfo().lastModifiedTime()); + } + + private void compare(ModelVersion left, ModelVersion right) { + Assertions.assertEquals(left.version(), right.version()); + Assertions.assertEquals(left.uri(), right.uri()); + Assertions.assertArrayEquals(left.aliases(), right.aliases()); + Assertions.assertEquals(left.comment(), right.comment()); + Assertions.assertEquals(left.properties(), right.properties()); + + Assertions.assertNotNull(right.auditInfo()); + Assertions.assertEquals(left.auditInfo().creator(), right.auditInfo().creator()); + Assertions.assertEquals(left.auditInfo().createTime(), right.auditInfo().createTime()); + Assertions.assertEquals(left.auditInfo().lastModifier(), right.auditInfo().lastModifier()); + Assertions.assertEquals( + left.auditInfo().lastModifiedTime(), right.auditInfo().lastModifiedTime()); + } +} From 132b468c4f41713b9463189c9ea00f89e84bf632 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 26 Dec 2024 15:50:06 +0800 Subject: [PATCH 37/47] [#5993] refactor: Move the JdbcAuthorizationPlugin to authorization-common module (#5994) ### What changes were proposed in this pull request? Move the JdbcAuthorizationPlugin to authorization-common module ### Why are the changes needed? Fix: #5993 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Just refactor. --- .../authorization-common/build.gradle.kts | 1 + .../common/AuthorizationProperties.java | 4 +- .../jdbc/JdbcAuthorizationPlugin.java | 3 +- .../JdbcAuthorizationProperties.java | 5 +- .../jdbc/JdbcAuthorizationSQL.java | 2 +- .../jdbc/JdbcMetadataObject.java | 0 .../authorization/jdbc/JdbcPrivilege.java | 0 .../jdbc/JdbcSecurableObject.java | 0 .../JdbcSecurableObjectMappingProvider.java | 0 .../jdbc/TestJdbcAuthorizationPlugin.java} | 3 +- .../authorization-jdbc/build.gradle.kts | 96 ------------------- settings.gradle.kts | 2 +- 12 files changed, 10 insertions(+), 106 deletions(-) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java (98%) rename authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/{common => jdbc}/JdbcAuthorizationProperties.java (92%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java (99%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java (100%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java (100%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java (100%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java (100%) rename authorizations/{authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java => authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java} (98%) delete mode 100644 authorizations/authorization-jdbc/build.gradle.kts diff --git a/authorizations/authorization-common/build.gradle.kts b/authorizations/authorization-common/build.gradle.kts index ba64510f2ce..9bab92dac3e 100644 --- a/authorizations/authorization-common/build.gradle.kts +++ b/authorizations/authorization-common/build.gradle.kts @@ -36,6 +36,7 @@ dependencies { } implementation(libs.bundles.log4j) implementation(libs.commons.lang3) + implementation(libs.commons.dbcp2) implementation(libs.guava) implementation(libs.javax.jaxb.api) { exclude("*") diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java index 3005cc5f3e9..3ece6353d6e 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java @@ -31,9 +31,9 @@ public AuthorizationProperties(Map properties) { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - abstract String getPropertiesPrefix(); + public abstract String getPropertiesPrefix(); - abstract void validate(); + public abstract void validate(); public static void validate(String type, Map properties) { switch (type) { diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java similarity index 98% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java index d9bc28636c3..cc3190413e1 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java @@ -40,7 +40,6 @@ import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; -import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.meta.AuditInfo; @@ -55,7 +54,7 @@ * JDBC-based authorization plugins can inherit this class and implement their own SQL statements. */ @Unstable -abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL { +public abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL { private static final String GROUP_PREFIX = "GRAVITINO_GROUP_"; private static final Logger LOG = LoggerFactory.getLogger(JdbcAuthorizationPlugin.class); diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java similarity index 92% rename from authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java index 9a5e7c6cc97..69a12135023 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.common; +package org.apache.gravitino.authorization.jdbc; import java.util.Map; +import org.apache.gravitino.authorization.common.AuthorizationProperties; /** The properties for JDBC authorization plugin. */ public class JdbcAuthorizationProperties extends AuthorizationProperties { @@ -39,7 +40,7 @@ private void check(String key, String errorMsg) { } @Override - String getPropertiesPrefix() { + public String getPropertiesPrefix() { return CONFIG_PREFIX; } diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java similarity index 99% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java index f7171ff354a..de031f70e78 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java @@ -25,7 +25,7 @@ /** Interface for SQL operations of the underlying access control system. */ @Unstable -interface JdbcAuthorizationSQL { +public interface JdbcAuthorizationSQL { /** * Get SQL statements for creating a user. diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java diff --git a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java similarity index 98% rename from authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java rename to authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java index e261fad78d2..ab91ba81e93 100644 --- a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java @@ -34,7 +34,6 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.User; -import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; import org.apache.gravitino.meta.RoleEntity; @@ -42,7 +41,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class JdbcAuthorizationPluginTest { +public class TestJdbcAuthorizationPlugin { private static List expectSQLs = Lists.newArrayList(); private static List expectTypes = Lists.newArrayList(); private static List expectObjectNames = Lists.newArrayList(); diff --git a/authorizations/authorization-jdbc/build.gradle.kts b/authorizations/authorization-jdbc/build.gradle.kts deleted file mode 100644 index 1a61f7c0cf9..00000000000 --- a/authorizations/authorization-jdbc/build.gradle.kts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -description = "authorization-jdbc" - -plugins { - `maven-publish` - id("java") - id("idea") -} - -dependencies { - implementation(project(":api")) { - exclude(group = "*") - } - implementation(project(":core")) { - exclude(group = "*") - } - implementation(project(":authorizations:authorization-common")) { - exclude(group = "*") - } - implementation(libs.bundles.log4j) - implementation(libs.commons.lang3) - implementation(libs.guava) - implementation(libs.javax.jaxb.api) { - exclude("*") - } - implementation(libs.javax.ws.rs.api) - implementation(libs.jettison) - compileOnly(libs.lombok) - implementation(libs.mail) - implementation(libs.rome) - implementation(libs.commons.dbcp2) - - testImplementation(project(":common")) - testImplementation(project(":clients:client-java")) - testImplementation(project(":server")) - testImplementation(project(":catalogs:catalog-common")) - testImplementation(project(":integration-test-common", "testArtifacts")) - testImplementation(libs.junit.jupiter.api) - testImplementation(libs.mockito.core) - testImplementation(libs.testcontainers) - testRuntimeOnly(libs.junit.jupiter.engine) -} - -tasks { - val runtimeJars by registering(Copy::class) { - from(configurations.runtimeClasspath) - into("build/libs") - } - - val copyAuthorizationLibs by registering(Copy::class) { - dependsOn("jar", runtimeJars) - from("build/libs") { - exclude("guava-*.jar") - exclude("log4j-*.jar") - exclude("slf4j-*.jar") - } - into("$rootDir/distribution/package/authorizations/ranger/libs") - } - - register("copyLibAndConfig", Copy::class) { - dependsOn(copyAuthorizationLibs) - } - - jar { - dependsOn(runtimeJars) - } -} - -tasks.test { - dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars") - - val skipITs = project.hasProperty("skipITs") - if (skipITs) { - // Exclude integration tests - exclude("**/integration/test/**") - } else { - dependsOn(tasks.jar) - } -} diff --git a/settings.gradle.kts b/settings.gradle.kts index f38443db206..562614764b3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,7 +57,7 @@ if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() == true) } include("iceberg:iceberg-common") include("iceberg:iceberg-rest-server") -include("authorizations:authorization-ranger", "authorizations:authorization-jdbc", "authorizations:authorization-common", "authorizations:authorization-chain") +include("authorizations:authorization-ranger", "authorizations:authorization-common", "authorizations:authorization-chain") include("trino-connector:trino-connector", "trino-connector:integration-test") include("spark-connector:spark-common") // kyuubi hive connector doesn't support 2.13 for Spark3.3 From 350d788345bdd5ea6929dc0eec9a35b81fd07252 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 26 Dec 2024 17:28:44 +0800 Subject: [PATCH 38/47] [#5993][FOLLOWUP] Make some methods public (#6002) ### What changes were proposed in this pull request? Make some methods public, let some classes to reuse some methods. ### Why are the changes needed? This is a follow-up PR. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? No need. Just refactor. --- .../authorization/jdbc/JdbcAuthorizationPlugin.java | 6 +++--- .../gravitino/authorization/jdbc/JdbcSecurableObject.java | 2 +- .../authorization/jdbc/TestJdbcAuthorizationPlugin.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java index cc3190413e1..d0a1b0897ec 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java @@ -350,11 +350,11 @@ public List getRevokeRoleSQL(String roleName, String revokerType, String } @VisibleForTesting - Connection getConnection() throws SQLException { + public Connection getConnection() throws SQLException { return dataSource.getConnection(); } - protected void executeUpdateSQL(String sql) { + public void executeUpdateSQL(String sql) { executeUpdateSQL(sql, null); } @@ -381,7 +381,7 @@ protected AuthorizationPluginException toAuthorizationPluginException(SQLExcepti "JDBC authorization plugin fail to execute SQL, error code: %d", se.getErrorCode()); } - void executeUpdateSQL(String sql, String ignoreErrorMsg) { + public void executeUpdateSQL(String sql, String ignoreErrorMsg) { try (final Connection connection = getConnection()) { try (final Statement statement = connection.createStatement()) { statement.executeUpdate(sql); diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java index 78b82e2a8da..2c721093e2c 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java @@ -44,7 +44,7 @@ private JdbcSecurableObject( this.privileges = privileges; } - static JdbcSecurableObject create( + public static JdbcSecurableObject create( String schema, String table, List privileges) { String parent = table == null ? null : schema; String name = table == null ? schema : table; diff --git a/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java index ab91ba81e93..6f9f7e29c37 100644 --- a/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java @@ -74,7 +74,7 @@ public List getSetOwnerSQL( return Collections.emptyList(); } - void executeUpdateSQL(String sql, String ignoreErrorMsg) { + public void executeUpdateSQL(String sql, String ignoreErrorMsg) { Assertions.assertEquals(expectSQLs.get(currentSQLIndex), sql); currentSQLIndex++; } From af433fd375ce2f2a1196064020f8a23196eda20b Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 26 Dec 2024 20:40:57 +0800 Subject: [PATCH 39/47] [#5987] improvement: Add more configurations for the config API (#5999) ### What changes were proposed in this pull request? Add more configurations for the config API ### Why are the changes needed? Fix #5987 Front end can use this config option to optimize the user experience. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? ![image](https://github.com/user-attachments/assets/a5f26c61-9e10-4dfa-bbcd-54cb887b1b71) --- .../gravitino/server/web/ConfigServlet.java | 10 ++-- .../server/web/TestConfigServlet.java | 46 +++++++++++++++++++ web/web/src/lib/store/auth/index.js | 2 +- 3 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 server/src/test/java/org/apache/gravitino/server/web/TestConfigServlet.java diff --git a/server/src/main/java/org/apache/gravitino/server/web/ConfigServlet.java b/server/src/main/java/org/apache/gravitino/server/web/ConfigServlet.java index cdce0a0b125..345a555cd9e 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/ConfigServlet.java +++ b/server/src/main/java/org/apache/gravitino/server/web/ConfigServlet.java @@ -42,22 +42,20 @@ public class ConfigServlet extends HttpServlet { ImmutableSet.of(OAuthConfig.DEFAULT_SERVER_URI, OAuthConfig.DEFAULT_TOKEN_PATH); private static final ImmutableSet> basicConfigEntries = - ImmutableSet.of(Configs.AUTHENTICATORS); + ImmutableSet.of(Configs.AUTHENTICATORS, Configs.ENABLE_AUTHORIZATION); - private final Map configs = Maps.newHashMap(); + private final Map configs = Maps.newHashMap(); public ConfigServlet(ServerConfig serverConfig) { for (ConfigEntry key : basicConfigEntries) { - String config = String.valueOf(serverConfig.get(key)); - configs.put(key.getKey(), config); + configs.put(key.getKey(), serverConfig.get(key)); } if (serverConfig .get(Configs.AUTHENTICATORS) .contains(AuthenticatorType.OAUTH.name().toLowerCase())) { for (ConfigEntry key : oauthConfigEntries) { - String config = String.valueOf(serverConfig.get(key)); - configs.put(key.getKey(), config); + configs.put(key.getKey(), serverConfig.get(key)); } } } diff --git a/server/src/test/java/org/apache/gravitino/server/web/TestConfigServlet.java b/server/src/test/java/org/apache/gravitino/server/web/TestConfigServlet.java new file mode 100644 index 00000000000..c76587d0397 --- /dev/null +++ b/server/src/test/java/org/apache/gravitino/server/web/TestConfigServlet.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.PrintWriter; +import javax.servlet.http.HttpServletResponse; +import org.apache.gravitino.server.ServerConfig; +import org.junit.jupiter.api.Test; + +public class TestConfigServlet { + + @Test + public void testConfigServlet() throws Exception { + ServerConfig serverConfig = new ServerConfig(); + ConfigServlet configServlet = new ConfigServlet(serverConfig); + configServlet.init(); + HttpServletResponse res = mock(HttpServletResponse.class); + PrintWriter writer = mock(PrintWriter.class); + when(res.getWriter()).thenReturn(writer); + configServlet.doGet(null, res); + verify(writer) + .write( + "{\"gravitino.authorization.enable\":false,\"gravitino.authenticators\":[\"simple\"]}"); + configServlet.destroy(); + } +} diff --git a/web/web/src/lib/store/auth/index.js b/web/web/src/lib/store/auth/index.js index 83d58394c90..ec0f1f52122 100644 --- a/web/web/src/lib/store/auth/index.js +++ b/web/web/src/lib/store/auth/index.js @@ -40,7 +40,7 @@ export const getAuthConfigs = createAsyncThunk('auth/getAuthConfigs', async () = oauthUrl = `${res['gravitino.authenticator.oauth.serverUri']}${res['gravitino.authenticator.oauth.tokenPath']}` // ** get the first authenticator from the response. response example: "[simple, oauth]" - authType = res['gravitino.authenticators'].slice(1, -1).split(',')[0].trim() + authType = res['gravitino.authenticators'][0].trim() localStorage.setItem('oauthUrl', oauthUrl) From b3475db3c964ce17a1fb66737b2f6ab2fc6703b7 Mon Sep 17 00:00:00 2001 From: FANNG Date: Fri, 27 Dec 2024 09:49:23 +0800 Subject: [PATCH 40/47] [#4398] feat(core): support credential cache for Gravitino server (#5995) ### What changes were proposed in this pull request? add credential cache for Gravitino server, not support for Iceberg rest server yet. ### Why are the changes needed? Fix: #4398 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? testing in local env, get credential from Gravitino server and see whether it's fetched from remote or local cache --- .../credential/CredentialConstants.java | 2 + .../gravitino/config/ConfigBuilder.java | 25 +++++ .../gravitino/connector/PropertyEntry.java | 24 +++++ .../credential/CatalogCredentialContext.java | 24 +++++ .../credential/CatalogCredentialManager.java | 21 +++- .../gravitino/credential/CredentialCache.java | 101 ++++++++++++++++++ .../credential/CredentialCacheKey.java | 64 +++++++++++ .../PathBasedCredentialContext.java | 33 ++++++ .../credential/config/CredentialConfig.java | 56 +++++++++- .../credential/TestCredentialCacheKey.java | 56 ++++++++++ 10 files changed, 399 insertions(+), 7 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialCache.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java create mode 100644 core/src/test/java/org/apache/gravitino/credential/TestCredentialCacheKey.java diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index d2753f24b5e..c766a86c141 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -22,6 +22,8 @@ public class CredentialConstants { public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; public static final String CREDENTIAL_PROVIDERS = "credential-providers"; + public static final String CREDENTIAL_CACHE_EXPIRE_RATIO = "credential-cache-expire-ratio"; + public static final String CREDENTIAL_CACHE_MAX_SIZE = "credential-cache-max-size"; public static final String S3_TOKEN_CREDENTIAL_PROVIDER = "s3-token"; public static final String S3_TOKEN_EXPIRE_IN_SECS = "s3-token-expire-in-secs"; diff --git a/core/src/main/java/org/apache/gravitino/config/ConfigBuilder.java b/core/src/main/java/org/apache/gravitino/config/ConfigBuilder.java index 148cda4fa70..d42f445e10a 100644 --- a/core/src/main/java/org/apache/gravitino/config/ConfigBuilder.java +++ b/core/src/main/java/org/apache/gravitino/config/ConfigBuilder.java @@ -170,6 +170,31 @@ public ConfigEntry longConf() { return conf; } + /** + * Creates a configuration entry for Double data type. + * + * @return The created ConfigEntry instance for Double data type. + */ + public ConfigEntry doubleConf() { + ConfigEntry conf = + new ConfigEntry<>(key, version, doc, alternatives, isPublic, isDeprecated); + Function func = + s -> { + if (s == null || s.isEmpty()) { + return null; + } else { + return Double.parseDouble(s); + } + }; + conf.setValueConverter(func); + + Function stringFunc = + t -> Optional.ofNullable(t).map(String::valueOf).orElse(null); + conf.setStringConverter(stringFunc); + + return conf; + } + /** * Creates a configuration entry for Boolean data type. * diff --git a/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java b/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java index b4c788a60d8..a32c2fff21d 100644 --- a/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java +++ b/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java @@ -29,6 +29,7 @@ @Getter public final class PropertyEntry { + private final String name; private final String description; private final boolean required; @@ -90,6 +91,7 @@ private PropertyEntry( } public static class Builder { + private String name; private String description; private boolean required; @@ -214,6 +216,28 @@ public static PropertyEntry longPropertyEntry( .build(); } + public static PropertyEntry doublePropertyEntry( + String name, + String description, + boolean required, + boolean immutable, + double defaultValue, + boolean hidden, + boolean reserved) { + return new Builder() + .withName(name) + .withDescription(description) + .withRequired(required) + .withImmutable(immutable) + .withJavaType(Double.class) + .withDefaultValue(defaultValue) + .withDecoder(Double::parseDouble) + .withEncoder(String::valueOf) + .withHidden(hidden) + .withReserved(reserved) + .build(); + } + public static PropertyEntry integerPropertyEntry( String name, String description, diff --git a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java index a39dbba01bd..6ac0c498c02 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java +++ b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java @@ -19,6 +19,7 @@ package org.apache.gravitino.credential; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; import javax.validation.constraints.NotNull; @@ -35,4 +36,27 @@ public CatalogCredentialContext(String userName) { public String getUserName() { return userName; } + + @Override + public int hashCode() { + return Objects.hashCode(userName); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !(o instanceof CatalogCredentialContext)) { + return false; + } + return Objects.equal(userName, ((CatalogCredentialContext) o).userName); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("User name: ").append(userName); + return stringBuilder.toString(); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java index 2fe6fedccd9..0e407a399b4 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java +++ b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java @@ -34,20 +34,21 @@ public class CatalogCredentialManager implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(CatalogCredentialManager.class); + private final CredentialCache credentialCache; + private final String catalogName; private final Map credentialProviders; public CatalogCredentialManager(String catalogName, Map catalogProperties) { this.catalogName = catalogName; this.credentialProviders = CredentialUtils.loadCredentialProviders(catalogProperties); + this.credentialCache = new CredentialCache(); + credentialCache.initialize(catalogProperties); } public Credential getCredential(String credentialType, CredentialContext context) { - // todo: add credential cache - Preconditions.checkState( - credentialProviders.containsKey(credentialType), - String.format("Credential %s not found", credentialType)); - return credentialProviders.get(credentialType).getCredential(context); + CredentialCacheKey credentialCacheKey = new CredentialCacheKey(credentialType, context); + return credentialCache.getCredential(credentialCacheKey, cacheKey -> doGetCredential(cacheKey)); } @Override @@ -67,4 +68,14 @@ public void close() { } }); } + + private Credential doGetCredential(CredentialCacheKey credentialCacheKey) { + String credentialType = credentialCacheKey.getCredentialType(); + CredentialContext context = credentialCacheKey.getCredentialContext(); + LOG.debug("Try get credential, credential type: {}, context: {}.", credentialType, context); + Preconditions.checkState( + credentialProviders.containsKey(credentialType), + String.format("Credential %s not found", credentialType)); + return credentialProviders.get(credentialType).getCredential(context); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialCache.java b/core/src/main/java/org/apache/gravitino/credential/CredentialCache.java new file mode 100644 index 00000000000..afbb09d50ed --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialCache.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Expiry; +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.apache.gravitino.credential.config.CredentialConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CredentialCache implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(CredentialCache.class); + + // Calculates the credential expire time in the cache. + static class CredentialExpireTimeCalculator implements Expiry { + + private double credentialCacheExpireRatio; + + public CredentialExpireTimeCalculator(double credentialCacheExpireRatio) { + this.credentialCacheExpireRatio = credentialCacheExpireRatio; + } + + // Set expire time after add a credential in the cache. + @Override + public long expireAfterCreate(T key, Credential credential, long currentTime) { + long credentialExpireTime = credential.expireTimeInMs(); + long timeToExpire = credentialExpireTime - System.currentTimeMillis(); + if (timeToExpire <= 0) { + return 0; + } + + timeToExpire = (long) (timeToExpire * credentialCacheExpireRatio); + return TimeUnit.MILLISECONDS.toNanos(timeToExpire); + } + + // Not change expire time after update credential, this should not happen. + @Override + public long expireAfterUpdate(T key, Credential value, long currentTime, long currentDuration) { + return currentDuration; + } + + // Not change expire time after read credential. + @Override + public long expireAfterRead(T key, Credential value, long currentTime, long currentDuration) { + return currentDuration; + } + } + + private Cache credentialCache; + + public void initialize(Map catalogProperties) { + CredentialConfig credentialConfig = new CredentialConfig(catalogProperties); + long cacheSize = credentialConfig.get(CredentialConfig.CREDENTIAL_CACHE_MAX_SIZE); + double cacheExpireRatio = credentialConfig.get(CredentialConfig.CREDENTIAL_CACHE_EXPIRE_RATIO); + + this.credentialCache = + Caffeine.newBuilder() + .expireAfter(new CredentialExpireTimeCalculator(cacheExpireRatio)) + .maximumSize(cacheSize) + .removalListener( + (cacheKey, credential, c) -> + LOG.debug("Credential expire, cache key: {}.", cacheKey)) + .build(); + } + + public Credential getCredential(T cacheKey, Function credentialSupplier) { + return credentialCache.get(cacheKey, key -> credentialSupplier.apply(cacheKey)); + } + + @Override + public void close() throws IOException { + if (credentialCache != null) { + credentialCache.invalidateAll(); + credentialCache = null; + } + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java b/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java new file mode 100644 index 00000000000..1d0d8f7b3b6 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.Objects; +import lombok.Getter; + +@Getter +public class CredentialCacheKey { + + private final String credentialType; + private final CredentialContext credentialContext; + + public CredentialCacheKey(String credentialType, CredentialContext credentialContext) { + this.credentialType = credentialType; + this.credentialContext = credentialContext; + } + + @Override + public int hashCode() { + return Objects.hash(credentialType, credentialContext); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !(o instanceof CredentialCacheKey)) { + return false; + } + CredentialCacheKey that = (CredentialCacheKey) o; + return Objects.equals(credentialType, that.credentialType) + && Objects.equals(credentialContext, that.credentialContext); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder + .append("credentialType: ") + .append(credentialType) + .append("credentialContext: ") + .append(credentialContext); + return stringBuilder.toString(); + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java b/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java index 03e7bbe0e31..06d17b134ba 100644 --- a/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java +++ b/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java @@ -20,6 +20,7 @@ package org.apache.gravitino.credential; import com.google.common.base.Preconditions; +import java.util.Objects; import java.util.Set; import javax.validation.constraints.NotNull; @@ -55,4 +56,36 @@ public Set getWritePaths() { public Set getReadPaths() { return readPaths; } + + @Override + public int hashCode() { + return Objects.hash(userName, writePaths, readPaths); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !(o instanceof PathBasedCredentialContext)) { + return false; + } + PathBasedCredentialContext that = (PathBasedCredentialContext) o; + return Objects.equals(userName, that.userName) + && Objects.equals(writePaths, that.writePaths) + && Objects.equals(readPaths, that.readPaths); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder + .append("User name: ") + .append(userName) + .append(", write path: ") + .append(writePaths) + .append(", read path: ") + .append(readPaths); + return stringBuilder.toString(); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java index d8823417cda..31a5183cc22 100644 --- a/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java +++ b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java @@ -21,16 +21,23 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; +import org.apache.gravitino.Config; +import org.apache.gravitino.config.ConfigBuilder; +import org.apache.gravitino.config.ConfigConstants; +import org.apache.gravitino.config.ConfigEntry; import org.apache.gravitino.connector.PropertyEntry; import org.apache.gravitino.credential.CredentialConstants; -public class CredentialConfig { +public class CredentialConfig extends Config { + + private static final long DEFAULT_CREDENTIAL_CACHE_MAX_SIZE = 10_000L; + private static final double DEFAULT_CREDENTIAL_CACHE_EXPIRE_RATIO = 0.15d; public static final Map> CREDENTIAL_PROPERTY_ENTRIES = new ImmutableMap.Builder>() .put( CredentialConstants.CREDENTIAL_PROVIDERS, - PropertyEntry.booleanPropertyEntry( + PropertyEntry.stringPropertyEntry( CredentialConstants.CREDENTIAL_PROVIDERS, "Credential providers for the Gravitino catalog, schema, fileset, table, etc.", false /* required */, @@ -38,5 +45,50 @@ public class CredentialConfig { null /* default value */, false /* hidden */, false /* reserved */)) + .put( + CredentialConstants.CREDENTIAL_CACHE_EXPIRE_RATIO, + PropertyEntry.doublePropertyEntry( + CredentialConstants.CREDENTIAL_CACHE_EXPIRE_RATIO, + "Ratio of the credential's expiration time when Gravitino remove credential from the cache.", + false /* required */, + false /* immutable */, + DEFAULT_CREDENTIAL_CACHE_EXPIRE_RATIO /* default value */, + false /* hidden */, + false /* reserved */)) + .put( + CredentialConstants.CREDENTIAL_CACHE_MAX_SIZE, + PropertyEntry.longPropertyEntry( + CredentialConstants.CREDENTIAL_CACHE_MAX_SIZE, + "Max size for the credential cache.", + false /* required */, + false /* immutable */, + DEFAULT_CREDENTIAL_CACHE_MAX_SIZE /* default value */, + false /* hidden */, + false /* reserved */)) .build(); + + public static final ConfigEntry CREDENTIAL_CACHE_EXPIRE_RATIO = + new ConfigBuilder(CredentialConstants.CREDENTIAL_CACHE_EXPIRE_RATIO) + .doc( + "Ratio of the credential's expiration time when Gravitino remove credential from the " + + "cache.") + .version(ConfigConstants.VERSION_0_8_0) + .doubleConf() + .checkValue( + ratio -> ratio >= 0 && ratio < 1, + "Ratio of the credential's expiration time should greater than or equal to 0 " + + "and less than 1.") + .createWithDefault(DEFAULT_CREDENTIAL_CACHE_EXPIRE_RATIO); + + public static final ConfigEntry CREDENTIAL_CACHE_MAX_SIZE = + new ConfigBuilder(CredentialConstants.CREDENTIAL_CACHE_MAX_SIZE) + .doc("Max cache size for the credential.") + .version(ConfigConstants.VERSION_0_8_0) + .longConf() + .createWithDefault(DEFAULT_CREDENTIAL_CACHE_MAX_SIZE); + + public CredentialConfig(Map properties) { + super(false); + loadFromMap(properties, k -> true); + } } diff --git a/core/src/test/java/org/apache/gravitino/credential/TestCredentialCacheKey.java b/core/src/test/java/org/apache/gravitino/credential/TestCredentialCacheKey.java new file mode 100644 index 00000000000..b29c660a97d --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/credential/TestCredentialCacheKey.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableSet; + +public class TestCredentialCacheKey { + + @Test + void testCredentialCacheKey() { + + PathBasedCredentialContext context = + new PathBasedCredentialContext("user1", ImmutableSet.of("path1"), ImmutableSet.of("path2")); + PathBasedCredentialContext contextWithDiffUser = + new PathBasedCredentialContext("user2", ImmutableSet.of("path1"), ImmutableSet.of("path2")); + PathBasedCredentialContext contextWithDiffPath = + new PathBasedCredentialContext("user1", ImmutableSet.of("path3"), ImmutableSet.of("path4")); + + CredentialCacheKey key1 = new CredentialCacheKey("s3-token", context); + + Set cache = ImmutableSet.of(key1); + Assertions.assertTrue(cache.contains(key1)); + + // different user + CredentialCacheKey key2 = new CredentialCacheKey("s3-token", contextWithDiffUser); + Assertions.assertFalse(cache.contains(key2)); + + // different path + CredentialCacheKey key3 = new CredentialCacheKey("s3-token", contextWithDiffPath); + Assertions.assertFalse(cache.contains(key3)); + + // different credential type + CredentialCacheKey key4 = new CredentialCacheKey("s3-token1", context); + Assertions.assertFalse(cache.contains(key4)); + } +} From 8db9b8a6a16dc0d5e593890df7848b4c64a8741a Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Fri, 27 Dec 2024 16:18:19 +0800 Subject: [PATCH 41/47] [#5585] improvement(bundles): Refactor bundle jars and provide core jars that does not contains hadoop-{aws,gcp,aliyun,azure} (#5806) ### What changes were proposed in this pull request? Provide another kind of bundle jars that does not contains hadoop-{aws,gcp,aliyun,azure} like aws-mini, gcp-mini. ### Why are the changes needed? To make it works in a wide range of Hadoop version Fix: #5585 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Existing UTs and ITs --- .../authorization-chain/build.gradle.kts | 10 +-- .../authorization-ranger/build.gradle.kts | 11 +-- build.gradle.kts | 8 +- bundles/aliyun-bundle/build.gradle.kts | 38 +++----- bundles/aliyun/build.gradle.kts | 87 +++++++++++++++++++ .../oss/credential/OSSSecretKeyProvider.java | 0 .../oss/credential/OSSTokenProvider.java | 0 .../oss/credential/policy/Condition.java | 0 .../oss/credential/policy/Effect.java | 0 .../oss/credential/policy/Policy.java | 0 .../oss/credential/policy/Statement.java | 0 .../oss/credential/policy/StringLike.java | 0 .../oss/fs/OSSFileSystemProvider.java | 0 ...itino.catalog.hadoop.fs.FileSystemProvider | 0 ...he.gravitino.credential.CredentialProvider | 0 bundles/aws-bundle/build.gradle.kts | 24 ++--- bundles/aws/build.gradle.kts | 68 +++++++++++++++ .../s3/credential/S3SecretKeyProvider.java | 0 .../s3/credential/S3TokenProvider.java | 0 .../gravitino/s3/fs/S3FileSystemProvider.java | 62 ++++++++++++- ...itino.catalog.hadoop.fs.FileSystemProvider | 0 ...he.gravitino.credential.CredentialProvider | 0 bundles/azure-bundle/build.gradle.kts | 25 ++---- bundles/azure/build.gradle.kts | 72 +++++++++++++++ .../abs/credential/ADLSLocationUtils.java | 0 .../abs/credential/ADLSTokenProvider.java | 0 .../credential/AzureAccountKeyProvider.java | 0 .../abs/fs/AzureFileSystemProvider.java | 0 ...itino.catalog.hadoop.fs.FileSystemProvider | 0 ...he.gravitino.credential.CredentialProvider | 0 bundles/gcp-bundle/build.gradle.kts | 24 ++--- .../org.apache.hadoop.fs.FileSystem | 0 bundles/gcp/build.gradle.kts | 69 +++++++++++++++ .../gcs/credential/GCSTokenProvider.java | 0 .../gcs/fs/GCSFileSystemProvider.java | 7 +- ...itino.catalog.hadoop.fs.FileSystemProvider | 0 ...he.gravitino.credential.CredentialProvider | 0 .../gravitino/catalog/hadoop/Constants.java | 26 ++++++ catalogs/catalog-hadoop/build.gradle.kts | 47 +++------- catalogs/catalog-hive/build.gradle.kts | 3 + catalogs/hadoop-common/build.gradle.kts | 5 +- .../catalog/hadoop/fs/FileSystemUtils.java | 6 +- .../build.gradle.kts | 3 + clients/filesystem-hadoop3/build.gradle.kts | 26 +++--- .../hadoop/GravitinoVirtualFileSystem.java | 24 +++++ docs/hadoop-catalog.md | 14 ++- docs/how-to-use-gvfs.md | 65 ++++++++++++-- gradle/libs.versions.toml | 12 ++- iceberg/iceberg-rest-server/build.gradle.kts | 8 +- integration-test-common/build.gradle.kts | 8 +- settings.gradle.kts | 10 +-- 51 files changed, 570 insertions(+), 192 deletions(-) create mode 100644 bundles/aliyun/build.gradle.kts rename bundles/{aliyun-bundle => aliyun}/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider (100%) rename bundles/{aliyun-bundle => aliyun}/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider (100%) create mode 100644 bundles/aws/build.gradle.kts rename bundles/{aws-bundle => aws}/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java (100%) rename bundles/{aws-bundle => aws}/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java (100%) rename bundles/{aws-bundle => aws}/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java (53%) rename bundles/{aws-bundle => aws}/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider (100%) rename bundles/{aws-bundle => aws}/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider (100%) create mode 100644 bundles/azure/build.gradle.kts rename bundles/{azure-bundle => azure}/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java (100%) rename bundles/{azure-bundle => azure}/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java (100%) rename bundles/{azure-bundle => azure}/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java (100%) rename bundles/{azure-bundle => azure}/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java (100%) rename bundles/{azure-bundle => azure}/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider (100%) rename bundles/{azure-bundle => azure}/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider (100%) rename bundles/gcp-bundle/src/main/resources/{META-INF/services => }/org.apache.hadoop.fs.FileSystem (100%) create mode 100644 bundles/gcp/build.gradle.kts rename bundles/{gcp-bundle => gcp}/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java (100%) rename bundles/{gcp-bundle => gcp}/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java (85%) rename bundles/{gcp-bundle => gcp}/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider (100%) rename bundles/{gcp-bundle => gcp}/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider (100%) create mode 100644 catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java rename catalogs/{catalog-hadoop => hadoop-common}/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java (95%) diff --git a/authorizations/authorization-chain/build.gradle.kts b/authorizations/authorization-chain/build.gradle.kts index d5cd160742c..e14cfa05ba9 100644 --- a/authorizations/authorization-chain/build.gradle.kts +++ b/authorizations/authorization-chain/build.gradle.kts @@ -81,6 +81,7 @@ dependencies { exclude("net.java.dev.jna") exclude("javax.ws.rs") exclude("org.eclipse.jetty") + exclude("org.apache.hadoop", "hadoop-common") } testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { @@ -93,11 +94,10 @@ dependencies { testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") { exclude("com.sun.jersey") } - testImplementation(libs.hadoop3.client) - testImplementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - } + + testImplementation(libs.hadoop3.client.api) + testImplementation(libs.hadoop3.client.runtime) + testImplementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts index d410b1ee8d4..8cc82250c23 100644 --- a/authorizations/authorization-ranger/build.gradle.kts +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -67,7 +67,12 @@ dependencies { exclude("net.java.dev.jna") exclude("javax.ws.rs") exclude("org.eclipse.jetty") + // Conflicts with hadoop-client-api used in hadoop-catalog. + exclude("org.apache.hadoop", "hadoop-common") } + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) + implementation(libs.rome) compileOnly(libs.lombok) testRuntimeOnly(libs.junit.jupiter.engine) @@ -92,11 +97,7 @@ dependencies { testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") { exclude("com.sun.jersey") } - testImplementation(libs.hadoop3.client) - testImplementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - } + testImplementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") diff --git a/build.gradle.kts b/build.gradle.kts index c64997f3a90..154b4e7f776 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -779,7 +779,7 @@ tasks { !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && it.name != "integration-test" && it.name != "bundled-catalog" && !it.name.startsWith("flink") && it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && - it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" && it.name != "hadoop-common" + it.parent?.name != "bundles" && it.name != "hadoop-common" ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -799,10 +799,8 @@ tasks { !it.name.startsWith("integration-test") && !it.name.startsWith("flink") && !it.name.startsWith("trino-connector") && - it.name != "bundled-catalog" && - it.name != "hive-metastore-common" && it.name != "gcp-bundle" && - it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" && - it.name != "hadoop-common" && it.name != "docs" + it.name != "hive-metastore-common" && + it.name != "docs" && it.name != "hadoop-common" && it.parent?.name != "bundles" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/bundles/aliyun-bundle/build.gradle.kts b/bundles/aliyun-bundle/build.gradle.kts index bc2d21a6851..c8377285599 100644 --- a/bundles/aliyun-bundle/build.gradle.kts +++ b/bundles/aliyun-bundle/build.gradle.kts @@ -25,32 +25,12 @@ plugins { } dependencies { - compileOnly(project(":api")) - compileOnly(project(":core")) - compileOnly(project(":catalogs:catalog-common")) - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(project(":catalogs:hadoop-common")) { - exclude("*") - } - compileOnly(libs.hadoop3.common) - - implementation(libs.aliyun.credentials.sdk) + implementation(project(":bundles:aliyun")) + implementation(libs.commons.collections3) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) implementation(libs.hadoop3.oss) - - // Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add - // https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1 - implementation(libs.sun.activation) - - // oss needs StringUtils from commons-lang3 or the following error will occur in 3.3.0 - // java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils - // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111) - // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323) - // org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611) - implementation(libs.commons.lang3) - - implementation(project(":catalogs:catalog-common")) { - exclude("*") - } + implementation(libs.httpclient) } tasks.withType(ShadowJar::class.java) { @@ -60,8 +40,12 @@ tasks.withType(ShadowJar::class.java) { mergeServiceFiles() // Relocate dependencies to avoid conflicts - relocate("org.jdom", "org.apache.gravitino.shaded.org.jdom") - relocate("org.apache.commons.lang3", "org.apache.gravitino.shaded.org.apache.commons.lang3") + relocate("org.jdom", "org.apache.gravitino.aliyun.shaded.org.jdom") + relocate("org.apache.commons.lang3", "org.apache.gravitino.aliyun.shaded.org.apache.commons.lang3") + relocate("com.fasterxml.jackson", "org.apache.gravitino.aliyun.shaded.com.fasterxml.jackson") + relocate("com.google.common", "org.apache.gravitino.aliyun.shaded.com.google.common") + relocate("org.apache.http", "org.apache.gravitino.aliyun.shaded.org.apache.http") + relocate("org.apache.commons.collections", "org.apache.gravitino.aliyun.shaded.org.apache.commons.collections") } tasks.jar { diff --git a/bundles/aliyun/build.gradle.kts b/bundles/aliyun/build.gradle.kts new file mode 100644 index 00000000000..f4d38d40b92 --- /dev/null +++ b/bundles/aliyun/build.gradle.kts @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":api")) + compileOnly(project(":catalogs:catalog-common")) + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":core")) + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) + compileOnly(libs.hadoop3.oss) + + implementation(project(":catalogs:catalog-common")) { + exclude("*") + } + implementation(project(":catalogs:hadoop-common")) { + exclude("*") + } + + implementation(libs.aliyun.credentials.sdk) + implementation(libs.commons.collections3) + + // oss needs StringUtils from commons-lang3 or the following error will occur in 3.3.0 + // java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils + // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111) + // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323) + // org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611) + implementation(libs.commons.lang3) + implementation(libs.guava) + + implementation(libs.httpclient) + implementation(libs.jackson.databind) + implementation(libs.jackson.annotations) + implementation(libs.jackson.datatype.jdk8) + implementation(libs.jackson.datatype.jsr310) + + // Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add + // https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1 + implementation(libs.sun.activation) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + mergeServiceFiles() + + // Relocate dependencies to avoid conflicts + relocate("org.jdom", "org.apache.gravitino.aliyun.shaded.org.jdom") + relocate("org.apache.commons.lang3", "org.apache.gravitino.aliyun.shaded.org.apache.commons.lang3") + relocate("com.fasterxml.jackson", "org.apache.gravitino.aliyun.shaded.com.fasterxml.jackson") + relocate("com.google.common", "org.apache.gravitino.aliyun.shaded.com.google.common") + relocate("org.apache.http", "org.apache.gravitino.aliyun.shaded.org.apache.http") + relocate("org.apache.commons.collections", "org.apache.gravitino.aliyun.shaded.org.apache.commons.collections") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java diff --git a/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/aliyun/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/aliyun/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/aliyun/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider similarity index 100% rename from bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider rename to bundles/aliyun/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts index 3af5c8b4f38..35b1e22a4f6 100644 --- a/bundles/aws-bundle/build.gradle.kts +++ b/bundles/aws-bundle/build.gradle.kts @@ -25,30 +25,20 @@ plugins { } dependencies { - compileOnly(project(":api")) - compileOnly(project(":core")) - compileOnly(project(":catalogs:catalog-common")) - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(project(":catalogs:hadoop-common")) { - exclude("*") - } - compileOnly(libs.hadoop3.common) - - implementation(libs.aws.iam) - implementation(libs.aws.policy) - implementation(libs.aws.sts) - implementation(libs.commons.lang3) + implementation(project(":bundles:aws")) implementation(libs.hadoop3.aws) - implementation(project(":catalogs:catalog-common")) { - exclude("*") - } + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) } tasks.withType(ShadowJar::class.java) { isZip64 = true configurations = listOf(project.configurations.runtimeClasspath.get()) - relocate("org.apache.commons", "org.apache.gravitino.aws.shaded.org.apache.commons") archiveClassifier.set("") + + relocate("org.apache.commons.lang3", "org.apache.gravitino.aws.shaded.org.apache.commons.lang3") + relocate("com.google.common", "org.apache.gravitino.aws.shaded.com.google.common") + relocate("com.fasterxml.jackson", "org.apache.gravitino.aws.shaded.com.fasterxml.jackson") } tasks.jar { diff --git a/bundles/aws/build.gradle.kts b/bundles/aws/build.gradle.kts new file mode 100644 index 00000000000..45fda5485d5 --- /dev/null +++ b/bundles/aws/build.gradle.kts @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":api")) + compileOnly(project(":catalogs:catalog-common")) + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":core")) + compileOnly(libs.hadoop3.aws) + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) + + implementation(project(":catalogs:catalog-common")) { + exclude("*") + } + implementation(project(":catalogs:hadoop-common")) { + exclude("*") + } + + implementation(libs.aws.iam) + implementation(libs.aws.policy) + implementation(libs.aws.sts) + implementation(libs.commons.lang3) + implementation(libs.hadoop3.aws) + implementation(libs.guava) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + + relocate("org.apache.commons.lang3", "org.apache.gravitino.aws.shaded.org.apache.commons.lang3") + relocate("com.google.common", "org.apache.gravitino.aws.shaded.com.google.common") + relocate("com.fasterxml.jackson", "org.apache.gravitino.aws.shaded.com.fasterxml.jackson") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java similarity index 100% rename from bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java rename to bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java similarity index 100% rename from bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java rename to bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java similarity index 53% rename from bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java rename to bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java index 0d755c1f564..b7cd569bbf6 100644 --- a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java +++ b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java @@ -19,9 +19,14 @@ package org.apache.gravitino.s3.fs; +import com.amazonaws.auth.AWSCredentialsProvider; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import java.io.IOException; +import java.util.List; import java.util.Map; import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; @@ -31,9 +36,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class S3FileSystemProvider implements FileSystemProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(S3FileSystemProvider.class); + @VisibleForTesting public static final Map GRAVITINO_KEY_TO_S3_HADOOP_KEY = ImmutableMap.of( @@ -41,20 +50,67 @@ public class S3FileSystemProvider implements FileSystemProvider { S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, Constants.ACCESS_KEY, S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, Constants.SECRET_KEY); + // We can't use Constants.AWS_CREDENTIALS_PROVIDER directly, as in 2.7, this key does not exist. + private static final String S3_CREDENTIAL_KEY = "fs.s3a.aws.credentials.provider"; + private static final String S3_SIMPLE_CREDENTIAL = + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"; + @Override public FileSystem getFileSystem(Path path, Map config) throws IOException { Configuration configuration = new Configuration(); Map hadoopConfMap = FileSystemUtils.toHadoopConfigMap(config, GRAVITINO_KEY_TO_S3_HADOOP_KEY); - if (!hadoopConfMap.containsKey(Constants.AWS_CREDENTIALS_PROVIDER)) { - configuration.set( - Constants.AWS_CREDENTIALS_PROVIDER, Constants.ASSUMED_ROLE_CREDENTIALS_DEFAULT); + if (!hadoopConfMap.containsKey(S3_CREDENTIAL_KEY)) { + hadoopConfMap.put(S3_CREDENTIAL_KEY, S3_SIMPLE_CREDENTIAL); } + hadoopConfMap.forEach(configuration::set); + + // Hadoop-aws 2 does not support IAMInstanceCredentialsProvider + checkAndSetCredentialProvider(configuration); + return S3AFileSystem.newInstance(path.toUri(), configuration); } + private void checkAndSetCredentialProvider(Configuration configuration) { + String provides = configuration.get(S3_CREDENTIAL_KEY); + if (provides == null) { + return; + } + + Splitter splitter = Splitter.on(',').trimResults().omitEmptyStrings(); + Joiner joiner = Joiner.on(",").skipNulls(); + // Split the list of providers + List providers = splitter.splitToList(provides); + List validProviders = Lists.newArrayList(); + + for (String provider : providers) { + try { + Class c = Class.forName(provider); + if (AWSCredentialsProvider.class.isAssignableFrom(c)) { + validProviders.add(provider); + } else { + LOGGER.warn( + "Credential provider {} is not a subclass of AWSCredentialsProvider, skipping", + provider); + } + } catch (Exception e) { + LOGGER.warn( + "Credential provider {} not found in the Hadoop runtime, falling back to default", + provider); + configuration.set(S3_CREDENTIAL_KEY, S3_SIMPLE_CREDENTIAL); + return; + } + } + + if (validProviders.isEmpty()) { + configuration.set(S3_CREDENTIAL_KEY, S3_SIMPLE_CREDENTIAL); + } else { + configuration.set(S3_CREDENTIAL_KEY, joiner.join(validProviders)); + } + } + /** * Get the scheme of the FileSystem. Attention, for S3 the schema is "s3a", not "s3". Users should * use "s3a://..." to access S3. diff --git a/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/aws/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/aws/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/aws/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider similarity index 100% rename from bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider rename to bundles/aws/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/bundles/azure-bundle/build.gradle.kts b/bundles/azure-bundle/build.gradle.kts index 9e4a4add54e..7d9e253ac8a 100644 --- a/bundles/azure-bundle/build.gradle.kts +++ b/bundles/azure-bundle/build.gradle.kts @@ -25,26 +25,10 @@ plugins { } dependencies { - compileOnly(project(":api")) - compileOnly(project(":core")) - compileOnly(project(":catalogs:catalog-common")) - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(project(":catalogs:hadoop-common")) { - exclude("*") - } - - compileOnly(libs.hadoop3.common) - - implementation(libs.azure.identity) - implementation(libs.azure.storage.file.datalake) - - implementation(libs.commons.lang3) - // runtime used - implementation(libs.commons.logging) + implementation(project(":bundles:azure")) implementation(libs.hadoop3.abs) - implementation(project(":catalogs:catalog-common")) { - exclude("*") - } + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) } tasks.withType(ShadowJar::class.java) { @@ -56,7 +40,8 @@ tasks.withType(ShadowJar::class.java) { relocate("org.apache.httpcomponents", "org.apache.gravitino.azure.shaded.org.apache.httpcomponents") relocate("org.apache.commons", "org.apache.gravitino.azure.shaded.org.apache.commons") relocate("com.fasterxml", "org.apache.gravitino.azure.shaded.com.fasterxml") - relocate("com.google.guava", "org.apache.gravitino.azure.shaded.com.google.guava") + relocate("com.google.common", "org.apache.gravitino.azure.shaded.com.google.common") + relocate("org.eclipse.jetty", "org.apache.gravitino.azure.shaded.org.eclipse.jetty") } tasks.jar { diff --git a/bundles/azure/build.gradle.kts b/bundles/azure/build.gradle.kts new file mode 100644 index 00000000000..59d8cf5f806 --- /dev/null +++ b/bundles/azure/build.gradle.kts @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":api")) + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":core")) + + compileOnly(libs.hadoop3.abs) + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) + + implementation(project(":catalogs:catalog-common")) { + exclude("*") + } + implementation(project(":catalogs:hadoop-common")) { + exclude("*") + } + + implementation(libs.azure.identity) + implementation(libs.azure.storage.file.datalake) + + implementation(libs.commons.lang3) + // runtime used + implementation(libs.commons.logging) + implementation(libs.guava) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + + // Relocate dependencies to avoid conflicts + relocate("org.apache.httpcomponents", "org.apache.gravitino.azure.shaded.org.apache.httpcomponents") + relocate("org.apache.commons", "org.apache.gravitino.azure.shaded.org.apache.commons") + relocate("com.fasterxml", "org.apache.gravitino.azure.shaded.com.fasterxml") + relocate("com.google.common", "org.apache.gravitino.azure.shaded.com.google.common") + relocate("org.eclipse.jetty", "org.apache.gravitino.azure.shaded.org.eclipse.jetty") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java diff --git a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/azure/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/azure/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/azure/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider similarity index 100% rename from bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider rename to bundles/azure/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/bundles/gcp-bundle/build.gradle.kts b/bundles/gcp-bundle/build.gradle.kts index bae7411c75e..73efaf9f22c 100644 --- a/bundles/gcp-bundle/build.gradle.kts +++ b/bundles/gcp-bundle/build.gradle.kts @@ -25,25 +25,10 @@ plugins { } dependencies { - compileOnly(project(":api")) - compileOnly(project(":core")) - compileOnly(project(":catalogs:catalog-common")) - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(project(":catalogs:hadoop-common")) { - exclude("*") - } - - compileOnly(libs.hadoop3.common) - - implementation(libs.commons.lang3) - // runtime used - implementation(libs.commons.logging) + implementation(project(":bundles:gcp")) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) implementation(libs.hadoop3.gcs) - implementation(project(":catalogs:catalog-common")) { - exclude("*") - } - implementation(libs.google.auth.http) - implementation(libs.google.auth.credentials) } tasks.withType(ShadowJar::class.java) { @@ -54,8 +39,9 @@ tasks.withType(ShadowJar::class.java) { // Relocate dependencies to avoid conflicts relocate("org.apache.httpcomponents", "org.apache.gravitino.gcp.shaded.org.apache.httpcomponents") relocate("org.apache.commons", "org.apache.gravitino.gcp.shaded.org.apache.commons") - relocate("com.google", "org.apache.gravitino.gcp.shaded.com.google") + relocate("com.google.common", "org.apache.gravitino.gcp.shaded.com.google.common") relocate("com.fasterxml", "org.apache.gravitino.gcp.shaded.com.fasterxml") + relocate("org.eclipse.jetty", "org.apache.gravitino.gcp.shaded.org.eclipse.jetty") } tasks.jar { diff --git a/bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/bundles/gcp-bundle/src/main/resources/org.apache.hadoop.fs.FileSystem similarity index 100% rename from bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem rename to bundles/gcp-bundle/src/main/resources/org.apache.hadoop.fs.FileSystem diff --git a/bundles/gcp/build.gradle.kts b/bundles/gcp/build.gradle.kts new file mode 100644 index 00000000000..6f21dc3d5af --- /dev/null +++ b/bundles/gcp/build.gradle.kts @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":api")) + compileOnly(project(":catalogs:catalog-common")) + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":core")) + + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) + + implementation(project(":catalogs:catalog-common")) { + exclude("*") + } + implementation(project(":catalogs:hadoop-common")) { + exclude("*") + } + implementation(libs.commons.lang3) + // runtime used + implementation(libs.commons.logging) + implementation(libs.google.auth.credentials) + implementation(libs.google.auth.http) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + + // Relocate dependencies to avoid conflicts + relocate("org.apache.httpcomponents", "org.apache.gravitino.gcp.shaded.org.apache.httpcomponents") + relocate("org.apache.commons", "org.apache.gravitino.gcp.shaded.org.apache.commons") + relocate("com.google.common", "org.apache.gravitino.gcp.shaded.com.google.common") + relocate("com.fasterxml", "org.apache.gravitino.gcp.shaded.com.fasterxml") + relocate("com.fasterxml.jackson", "org.apache.gravitino.gcp.shaded.com.fasterxml.jackson") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java similarity index 100% rename from bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java rename to bundles/gcp/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java diff --git a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java similarity index 85% rename from bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java rename to bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java index a07ff3d6ece..0055e167c49 100644 --- a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java +++ b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.gcs.fs; -import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import java.io.IOException; @@ -29,11 +28,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class GCSFileSystemProvider implements FileSystemProvider { - private static final Logger LOGGER = LoggerFactory.getLogger(GCSFileSystemProvider.class); private static final String GCS_SERVICE_ACCOUNT_JSON_FILE = "fs.gs.auth.service.account.json.keyfile"; @@ -46,8 +42,7 @@ public FileSystem getFileSystem(Path path, Map config) throws IO Configuration configuration = new Configuration(); FileSystemUtils.toHadoopConfigMap(config, GRAVITINO_KEY_TO_GCS_HADOOP_KEY) .forEach(configuration::set); - LOGGER.info("Creating GCS file system with config: {}", config); - return GoogleHadoopFileSystem.newInstance(path.toUri(), configuration); + return FileSystem.newInstance(path.toUri(), configuration); } @Override diff --git a/bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/gcp/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/gcp/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/gcp/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider similarity index 100% rename from bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider rename to bundles/gcp/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java new file mode 100644 index 00000000000..468728362bb --- /dev/null +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.catalog.hadoop; + +public class Constants { + + public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local"; + public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs"; +} diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 8873b795046..d599a5e72f1 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -28,43 +28,22 @@ dependencies { implementation(project(":api")) { exclude(group = "*") } - - implementation(project(":core")) { - exclude(group = "*") - } - - implementation(project(":common")) { - exclude(group = "*") - } - implementation(project(":catalogs:catalog-common")) { exclude(group = "*") } - implementation(project(":catalogs:hadoop-common")) { exclude(group = "*") } - - implementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - exclude("org.eclipse.jetty", "*") - exclude("org.apache.hadoop", "hadoop-auth") - exclude("org.apache.curator", "curator-client") - exclude("org.apache.curator", "curator-framework") - exclude("org.apache.curator", "curator-recipes") - exclude("org.apache.avro", "avro") - exclude("com.sun.jersey", "jersey-servlet") + implementation(project(":common")) { + exclude(group = "*") } - - implementation(libs.hadoop3.client) { - exclude("org.apache.hadoop", "hadoop-mapreduce-client-core") - exclude("org.apache.hadoop", "hadoop-mapreduce-client-jobclient") - exclude("org.apache.hadoop", "hadoop-yarn-api") - exclude("org.apache.hadoop", "hadoop-yarn-client") - exclude("com.squareup.okhttp", "okhttp") + implementation(project(":core")) { + exclude(group = "*") } - + implementation(libs.commons.lang3) + implementation(libs.commons.io) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) implementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") @@ -74,20 +53,18 @@ dependencies { exclude("io.netty") exclude("org.fusesource.leveldbjni") } - implementation(libs.slf4j.api) compileOnly(libs.guava) - testImplementation(project(":bundles:aws-bundle")) - testImplementation(project(":bundles:gcp-bundle")) - testImplementation(project(":bundles:aliyun-bundle")) - testImplementation(project(":bundles:azure-bundle")) testImplementation(project(":clients:client-java")) + testImplementation(project(":bundles:aws-bundle", configuration = "shadow")) + testImplementation(project(":bundles:gcp-bundle", configuration = "shadow")) + testImplementation(project(":bundles:aliyun-bundle", configuration = "shadow")) + testImplementation(project(":bundles:azure-bundle", configuration = "shadow")) testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(project(":server")) testImplementation(project(":server-common")) - testImplementation(libs.bundles.log4j) testImplementation(libs.hadoop3.gcs) testImplementation(libs.hadoop3.minicluster) diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index b471fccead1..6a8b815ab97 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -96,6 +96,9 @@ dependencies { testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(project(":server")) testImplementation(project(":server-common")) + testImplementation(project(":catalogs:hadoop-common")) { + exclude("*") + } testImplementation(libs.bundles.jetty) testImplementation(libs.bundles.jersey) diff --git a/catalogs/hadoop-common/build.gradle.kts b/catalogs/hadoop-common/build.gradle.kts index ab768cb1f11..566ce5986e3 100644 --- a/catalogs/hadoop-common/build.gradle.kts +++ b/catalogs/hadoop-common/build.gradle.kts @@ -23,6 +23,9 @@ plugins { // try to avoid adding extra dependencies because it is used by catalogs and connectors. dependencies { + implementation(project(":catalogs:catalog-common")) implementation(libs.commons.lang3) - implementation(libs.hadoop3.common) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) + implementation(libs.guava) } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java similarity index 95% rename from catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java rename to catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 129a8e88274..a1434e85c3e 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.catalog.hadoop.fs; -import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_HDFS_FS_PROVIDER; -import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_LOCAL_FS_PROVIDER; +import static org.apache.gravitino.catalog.hadoop.Constants.BUILTIN_HDFS_FS_PROVIDER; +import static org.apache.gravitino.catalog.hadoop.Constants.BUILTIN_LOCAL_FS_PROVIDER; import static org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider.GRAVITINO_BYPASS; import com.google.common.collect.Maps; @@ -45,7 +45,7 @@ public static Map getFileSystemProviders(String file fileSystemProviders != null ? Arrays.stream(fileSystemProviders.split(",")) .map(f -> f.trim().toLowerCase(Locale.ROOT)) - .collect(java.util.stream.Collectors.toSet()) + .collect(Collectors.toSet()) : Sets.newHashSet(); // Add built-in file system providers to the use list automatically. diff --git a/clients/filesystem-hadoop3-runtime/build.gradle.kts b/clients/filesystem-hadoop3-runtime/build.gradle.kts index 8081a55604e..db439a4981e 100644 --- a/clients/filesystem-hadoop3-runtime/build.gradle.kts +++ b/clients/filesystem-hadoop3-runtime/build.gradle.kts @@ -28,6 +28,7 @@ plugins { dependencies { implementation(project(":clients:filesystem-hadoop3")) implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(libs.commons.lang3) } tasks.withType(ShadowJar::class.java) { @@ -38,6 +39,8 @@ tasks.withType(ShadowJar::class.java) { // Relocate dependencies to avoid conflicts relocate("com.google", "org.apache.gravitino.shaded.com.google") relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine") + // relocate common lang3 package + relocate("org.apache.commons.lang3", "org.apache.gravitino.shaded.org.apache.commons.lang3") } tasks.jar { diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts index d24eb4efdf2..424f6a11406 100644 --- a/clients/filesystem-hadoop3/build.gradle.kts +++ b/clients/filesystem-hadoop3/build.gradle.kts @@ -25,7 +25,8 @@ plugins { dependencies { compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) - compileOnly(libs.hadoop3.common) + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) implementation(project(":catalogs:catalog-common")) { exclude(group = "*") @@ -35,32 +36,31 @@ dependencies { } implementation(libs.caffeine) + implementation(libs.guava) + implementation(libs.commons.lang3) testImplementation(project(":api")) testImplementation(project(":core")) + testImplementation(project(":catalogs:catalog-hadoop")) testImplementation(project(":common")) testImplementation(project(":server")) testImplementation(project(":server-common")) testImplementation(project(":clients:client-java")) testImplementation(project(":integration-test-common", "testArtifacts")) - testImplementation(project(":catalogs:catalog-hadoop")) - testImplementation(project(":bundles:gcp-bundle")) - testImplementation(project(":bundles:aliyun-bundle")) - testImplementation(project(":bundles:aws-bundle")) - testImplementation(project(":bundles:azure-bundle")) - testImplementation(project(":bundles:gcp-bundle")) + + testImplementation(project(":bundles:aws-bundle", configuration = "shadow")) + testImplementation(project(":bundles:gcp-bundle", configuration = "shadow")) + testImplementation(project(":bundles:aliyun-bundle", configuration = "shadow")) + testImplementation(project(":bundles:azure-bundle", configuration = "shadow")) testImplementation(libs.awaitility) testImplementation(libs.bundles.jetty) testImplementation(libs.bundles.jersey) testImplementation(libs.bundles.jwt) - testImplementation(libs.guava) - testImplementation(libs.hadoop3.client) - testImplementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - } + testImplementation(libs.hadoop3.client.api) + testImplementation(libs.hadoop3.client.runtime) + testImplementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index e18e376b46c..a9c40e55840 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -40,6 +40,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.audit.CallerContext; import org.apache.gravitino.audit.FilesetAuditConstants; @@ -392,6 +393,11 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat scheme, GravitinoVirtualFileSystemConfiguration.GVFS_SCHEME); } + // Reset the FileSystem service loader to make sure the FileSystem will reload the + // service file systems, this is a temporary solution to fix the issue + // https://github.com/apache/gravitino/issues/5609 + resetFileSystemServiceLoader(scheme); + Map maps = getConfigMap(getConf()); return provider.getFileSystem(filePath, maps); } catch (IOException ioe) { @@ -404,6 +410,24 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat return new FilesetContextPair(new Path(actualFileLocation), fs); } + private void resetFileSystemServiceLoader(String fsScheme) { + try { + Map> serviceFileSystems = + (Map>) + FieldUtils.getField(FileSystem.class, "SERVICE_FILE_SYSTEMS", true).get(null); + + if (serviceFileSystems.containsKey(fsScheme)) { + return; + } + + // Set this value to false so that FileSystem will reload the service file systems when + // needed. + FieldUtils.getField(FileSystem.class, "FILE_SYSTEMS_LOADED", true).set(null, false); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private Map getConfigMap(Configuration configuration) { Map maps = Maps.newHashMap(); configuration.forEach(entry -> maps.put(entry.getKey(), entry.getValue())); diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index ce58826cb93..9048556ffa5 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -10,10 +10,8 @@ license: "This software is licensed under the Apache License version 2." Hadoop catalog is a fileset catalog that using Hadoop Compatible File System (HCFS) to manage the storage location of the fileset. Currently, it supports local filesystem and HDFS. For -object storage like S3, GCS, and Azure Blob Storage, you can put the hadoop object store jar like -hadoop-aws into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory to enable the support. -Gravitino itself hasn't yet tested the object storage support, so if you have any issue, -please create an [issue](https://github.com/apache/gravitino/issues). +object storage like S3, GCS, Azure Blob Storage and OSS, you can put the hadoop object store jar like +`gravitino-aws-bundle-{gravitino-version}.jar` into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory to enable the support. Note that Gravitino uses Hadoop 3 dependencies to build Hadoop catalog. Theoretically, it should be compatible with both Hadoop 2.x and 3.x, since Gravitino doesn't leverage any new features in @@ -52,7 +50,7 @@ Apart from the above properties, to access fileset like HDFS, S3, GCS, OSS or cu | `s3-access-key-id` | The access key of the AWS S3. | (none) | Yes if it's a S3 fileset. | 0.7.0-incubating | | `s3-secret-access-key` | The secret key of the AWS S3. | (none) | Yes if it's a S3 fileset. | 0.7.0-incubating | -At the same time, you need to place the corresponding bundle jar [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aws-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. +At the same time, you need to place the corresponding bundle jar [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. #### GCS fileset @@ -62,7 +60,7 @@ At the same time, you need to place the corresponding bundle jar [`gravitino-aws | `default-filesystem-provider` | The name default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `builtin-local`, for GCS, if we set this value, we can omit the prefix 'gs://' in the location. | `builtin-local` | No | 0.7.0-incubating | | `gcs-service-account-file` | The path of GCS service account JSON file. | (none) | Yes if it's a GCS fileset. | 0.7.0-incubating | -In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gcp-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. +In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-gcp-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. #### OSS fileset @@ -74,7 +72,7 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp- | `oss-access-key-id` | The access key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset. | 0.7.0-incubating | | `oss-secret-access-key` | The secret key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset. | 0.7.0-incubating | -In the meantime, you need to place the corresponding bundle jar [`gravitino-aliyun-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aliyun-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. +In the meantime, you need to place the corresponding bundle jar [`gravitino-aliyun-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aliyun-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. #### Azure Blob Storage fileset @@ -86,7 +84,7 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-aliy | `azure-storage-account-name ` | The account name of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | | `azure-storage-account-key` | The account key of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | -Similar to the above, you need to place the corresponding bundle jar [`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/azure-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. +Similar to the above, you need to place the corresponding bundle jar [`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-azure-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. :::note - Gravitino contains builtin file system providers for local file system(`builtin-local`) and HDFS(`builtin-hdfs`), that is to say if `filesystem-providers` is not set, Gravitino will still support local file system and HDFS. Apart from that, you can set the `filesystem-providers` to support other file systems like S3, GCS, OSS or custom file system. diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md index 162d535be11..0dbfd867a3d 100644 --- a/docs/how-to-use-gvfs.md +++ b/docs/how-to-use-gvfs.md @@ -77,7 +77,9 @@ Apart from the above properties, to access fileset like S3, GCS, OSS and custom | `s3-access-key-id` | The access key of the AWS S3. | (none) | Yes if it's a S3 fileset.| 0.7.0-incubating | | `s3-secret-access-key` | The secret key of the AWS S3. | (none) | Yes if it's a S3 fileset.| 0.7.0-incubating | -At the same time, you need to place the corresponding bundle jar [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aws-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). +At the same time, you need to add the corresponding bundle jar +1. [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws-bundle/) in the classpath if no hadoop environment is available, or +2. [`gravitino-aws-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws/) and hadoop-aws jar and other necessary dependencies in the classpath. #### GCS fileset @@ -86,7 +88,9 @@ At the same time, you need to place the corresponding bundle jar [`gravitino-aws |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------|------------------| | `gcs-service-account-file` | The path of GCS service account JSON file. | (none) | Yes if it's a GCS fileset.| 0.7.0-incubating | -In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gcp-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). +In the meantime, you need to add the corresponding bundle jar +1. [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-gcp-bundle/) in the classpath if no hadoop environment is available, or +2. or [`gravitino-gcp-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-gcp/) and [gcs-connector jar](https://github.com/GoogleCloudDataproc/hadoop-connectors/releases) and other necessary dependencies in the classpath. #### OSS fileset @@ -97,7 +101,10 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp- | `oss-access-key-id` | The access key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset.| 0.7.0-incubating | | `oss-secret-access-key` | The secret key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset.| 0.7.0-incubating | -In the meantime, you need to place the corresponding bundle jar [`gravitino-aliyun-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aliyun-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). + +In the meantime, you need to place the corresponding bundle jar +1. [`gravitino-aliyun-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aliyun-bundle/) in the classpath if no hadoop environment is available, or +2. [`gravitino-aliyun-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aliyun/) and hadoop-aliyun jar and other necessary dependencies in the classpath. #### Azure Blob Storage fileset @@ -106,7 +113,9 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-aliy | `azure-storage-account-name` | The account name of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | | `azure-storage-account-key` | The account key of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | -Similar to the above, you need to place the corresponding bundle jar [`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/azure-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). +Similar to the above, you need to place the corresponding bundle jar +1. [`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-azure-bundle/) in the classpath if no hadoop environment is available, or +2. [`gravitino-azure-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-azure/) and hadoop-azure jar and other necessary dependencies in the classpath. #### Custom fileset Since 0.7.0-incubating, users can define their own fileset type and configure the corresponding properties, for more, please refer to [Custom Fileset](./hadoop-catalog.md#how-to-custom-your-own-hcfs-file-system-fileset). @@ -137,8 +146,13 @@ You can configure these properties in two ways: ``` :::note -If you want to access the S3, GCS, OSS or custom fileset through GVFS, apart from the above properties, you need to place the corresponding bundle jar in the Hadoop environment. -For example if you want to access the S3 fileset, you need to place the S3 bundle jar [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aws-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`) or add it to the classpath. +If you want to access the S3, GCS, OSS or custom fileset through GVFS, apart from the above properties, you need to place the corresponding bundle jars in the Hadoop environment. +For example, if you want to access the S3 fileset, you need to place +1. The aws hadoop bundle jar [`gravitino-aws-bundle-${gravitino-version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws-bundle/) +2. or [`gravitino-aws-${gravitino-version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws/), and hadoop-aws jar and other necessary dependencies + +to the classpath, it typically locates in `${HADOOP_HOME}/share/hadoop/common/lib/`). + ::: 2. Configure the properties in the `core-site.xml` file of the Hadoop environment: @@ -212,6 +226,12 @@ cp gravitino-filesystem-hadoop3-runtime-{version}.jar ${HADOOP_HOME}/share/hadoo # You need to ensure that the Kerberos has permission on the HDFS directory. kinit -kt your_kerberos.keytab your_kerberos@xxx.com + +# 4. Copy other dependencies to the Hadoop environment if you want to access the S3 fileset via GVFS +cp bundles/aws-bundle/build/libs/gravitino-aws-bundle-{version}.jar ${HADOOP_HOME}/share/hadoop/common/lib/ +cp clients/filesystem-hadoop3-runtime/build/libs/gravitino-filesystem-hadoop3-runtime-{version}-SNAPSHOT.jar ${HADOOP_HOME}/share/hadoop/common/lib/ +cp ${HADOOP_HOME}/share/hadoop/tools/lib/* ${HADOOP_HOME}/share/hadoop/common/lib/ + # 4. Try to list the fileset ./${HADOOP_HOME}/bin/hadoop dfs -ls gvfs://fileset/test_catalog/test_schema/test_fileset_1 ``` @@ -222,6 +242,36 @@ You can also perform operations on the files or directories managed by fileset t Make sure that your code is using the correct Hadoop environment, and that your environment has the `gravitino-filesystem-hadoop3-runtime-{version}.jar` dependency. +```xml + + + org.apache.gravitino + filesystem-hadoop3-runtime + {gravitino-version} + + + + + org.apache.gravitino + gravitino-aws-bundle + {gravitino-version} + + + + + org.apache.gravitino + gravitino-aws + {gravitino-version} + + + + org.apache.hadoop + hadoop-aws + {hadoop-version} + + +``` + For example: ```java @@ -462,8 +512,7 @@ from gravitino import gvfs options = { "cache_size": 20, "cache_expired_time": 3600, - "auth_type": "simple" - + "auth_type": "simple", # Optional, the following properties are required if you want to access the S3 fileset via GVFS python client, for GCS and OSS fileset, you should set the corresponding properties. "s3_endpoint": "http://localhost:9000", "s3_access_key_id": "minio", diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a33c300ee88..52bccd9b480 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -36,12 +36,13 @@ airlift-json = "237" airlift-resolver = "1.6" hive2 = "2.3.9" hadoop2 = "2.10.2" -hadoop3 = "3.3.0" +hadoop3 = "3.3.1" hadoop3-gcs = "1.9.4-hadoop3" -hadoop3-abs = "3.3.0" -hadoop3-aliyun = "3.3.0" -hadoop-minikdc = "3.3.0" +hadoop3-abs = "3.3.1" +hadoop3-aliyun = "3.3.1" +hadoop-minikdc = "3.3.1" htrace-core4 = "4.1.0-incubating" +httpclient = "4.4.1" httpclient5 = "5.2.1" mockserver = "5.15.0" commons-csv = "1.12.0" @@ -177,6 +178,8 @@ hadoop3-aws = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref = hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop3" } hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop3"} hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"} +hadoop3-client-api = { group = "org.apache.hadoop", name = "hadoop-client-api", version.ref = "hadoop3"} +hadoop3-client-runtime = { group = "org.apache.hadoop", name = "hadoop-client-runtime", version.ref = "hadoop3"} hadoop3-minicluster = { group = "org.apache.hadoop", name = "hadoop-minicluster", version.ref = "hadoop-minikdc"} hadoop3-gcs = { group = "com.google.cloud.bigdataoss", name = "gcs-connector", version.ref = "hadoop3-gcs"} hadoop3-oss = { group = "org.apache.hadoop", name = "hadoop-aliyun", version.ref = "hadoop3-aliyun"} @@ -184,6 +187,7 @@ hadoop3-abs = { group = "org.apache.hadoop", name = "hadoop-azure", version.ref htrace-core4 = { group = "org.apache.htrace", name = "htrace-core4", version.ref = "htrace-core4" } airlift-json = { group = "io.airlift", name = "json", version.ref = "airlift-json"} airlift-resolver = { group = "io.airlift.resolver", name = "resolver", version.ref = "airlift-resolver"} +httpclient = { group = "org.apache.httpcomponents", name = "httpclient", version.ref = "httpclient" } httpclient5 = { group = "org.apache.httpcomponents.client5", name = "httpclient5", version.ref = "httpclient5" } mockserver-netty = { group = "org.mock-server", name = "mockserver-netty", version.ref = "mockserver" } mockserver-client-java = { group = "org.mock-server", name = "mockserver-client-java", version.ref = "mockserver" } diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index 03fe32c92a9..fe35c4e7789 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -62,10 +62,10 @@ dependencies { annotationProcessor(libs.lombok) compileOnly(libs.lombok) - testImplementation(project(":bundles:aliyun-bundle")) - testImplementation(project(":bundles:aws-bundle")) - testImplementation(project(":bundles:gcp-bundle", configuration = "shadow")) - testImplementation(project(":bundles:azure-bundle")) + testImplementation(project(":bundles:aliyun")) + testImplementation(project(":bundles:aws")) + testImplementation(project(":bundles:gcp", configuration = "shadow")) + testImplementation(project(":bundles:azure", configuration = "shadow")) testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion") diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index 283169a76a9..bd15dc2a34f 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -53,11 +53,11 @@ dependencies { exclude("org.elasticsearch") exclude("org.elasticsearch.client") exclude("org.elasticsearch.plugin") + exclude("org.apache.hadoop", "hadoop-common") } - testImplementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - } + testImplementation(libs.hadoop3.client.api) + testImplementation(libs.hadoop3.client.runtime) + testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") } diff --git a/settings.gradle.kts b/settings.gradle.kts index 562614764b3..c865e14e7a2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -77,8 +77,8 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") -include(":bundles:aws-bundle") -include(":bundles:gcp-bundle") -include(":bundles:aliyun-bundle") -include(":bundles:azure-bundle") -include("catalogs:hadoop-common") +include(":bundles:aws", ":bundles:aws-bundle") +include(":bundles:gcp", ":bundles:gcp-bundle") +include(":bundles:aliyun", ":bundles:aliyun-bundle") +include(":bundles:azure", ":bundles:azure-bundle") +include(":catalogs:hadoop-common") From 5357dc4c53ffc937d30f9f74d79990a9bbc6427c Mon Sep 17 00:00:00 2001 From: FANNG Date: Sat, 28 Dec 2024 17:00:25 +0800 Subject: [PATCH 42/47] [#5989] The credential type of ADLSTokenCredentialProvider is not equal to the credential type of ADLSTokenCredential (#5990) ### What changes were proposed in this pull request? ADLSTokenCredentialProvider use the credential type from ADLSTokenCredential ### Why are the changes needed? Fix: #5989 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? add test --- .../credential/ADLSTokenCredential.java | 6 +-- bundles/aliyun/build.gradle.kts | 6 +++ .../credential/TestCredentialProvider.java | 40 +++++++++++++++++++ bundles/aws/build.gradle.kts | 6 +++ .../s3/credential/TestCredentialProvider.java | 39 ++++++++++++++++++ bundles/azure/build.gradle.kts | 6 +++ .../abs/credential/ADLSTokenProvider.java | 3 +- .../credential/AzureAccountKeyProvider.java | 3 +- .../credential/TestCredentialProvider.java | 40 +++++++++++++++++++ bundles/gcp/build.gradle.kts | 6 +++ .../gcs/credential/GCSTokenProvider.java | 3 +- .../credential/TestCredentialProvider.java | 34 ++++++++++++++++ .../credential/CredentialConstants.java | 9 ----- .../api/credential/adls_token_credential.py | 4 +- .../gravitino/utils/credential_factory.py | 2 +- .../unittests/test_credential_factory.py | 2 +- .../credential/TestCredentialFactory.java | 6 +-- .../test/IcebergRESTADLSTokenIT.java | 3 +- .../test/IcebergRESTAzureAccountKeyIT.java | 3 +- .../integration/test/IcebergRESTGCSIT.java | 3 +- .../integration/test/IcebergRESTOSSIT.java | 3 +- .../integration/test/IcebergRESTS3IT.java | 3 +- 22 files changed, 199 insertions(+), 31 deletions(-) create mode 100644 bundles/aliyun/src/test/java/org/apache/gravitino/oss/credential/TestCredentialProvider.java create mode 100644 bundles/aws/src/test/java/org/apache/gravitino/s3/credential/TestCredentialProvider.java create mode 100644 bundles/azure/src/test/java/org/apache/gravitino/abs/credential/TestCredentialProvider.java create mode 100644 bundles/gcp/src/test/java/org/apache/gravitino/gcs/credential/TestCredentialProvider.java diff --git a/api/src/main/java/org/apache/gravitino/credential/ADLSTokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/ADLSTokenCredential.java index 25c83c2f7cc..249b0ac0b03 100644 --- a/api/src/main/java/org/apache/gravitino/credential/ADLSTokenCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/ADLSTokenCredential.java @@ -27,8 +27,8 @@ /** ADLS SAS token credential. */ public class ADLSTokenCredential implements Credential { - /** ADLS SAS token credential type. */ - public static final String ADLS_SAS_TOKEN_CREDENTIAL_TYPE = "adls-sas-token"; + /** ADLS token credential type. */ + public static final String ADLS_TOKEN_CREDENTIAL_TYPE = "adls-token"; /** ADLS base domain */ public static final String ADLS_DOMAIN = "dfs.core.windows.net"; /** ADLS storage account name */ @@ -62,7 +62,7 @@ public ADLSTokenCredential() {} @Override public String credentialType() { - return ADLS_SAS_TOKEN_CREDENTIAL_TYPE; + return ADLS_TOKEN_CREDENTIAL_TYPE; } @Override diff --git a/bundles/aliyun/build.gradle.kts b/bundles/aliyun/build.gradle.kts index f4d38d40b92..9dfab9d6798 100644 --- a/bundles/aliyun/build.gradle.kts +++ b/bundles/aliyun/build.gradle.kts @@ -60,6 +60,12 @@ dependencies { // Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add // https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1 implementation(libs.sun.activation) + + testImplementation(project(":api")) + testImplementation(project(":core")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testRuntimeOnly(libs.junit.jupiter.engine) } tasks.withType(ShadowJar::class.java) { diff --git a/bundles/aliyun/src/test/java/org/apache/gravitino/oss/credential/TestCredentialProvider.java b/bundles/aliyun/src/test/java/org/apache/gravitino/oss/credential/TestCredentialProvider.java new file mode 100644 index 00000000000..0e8cd2fb748 --- /dev/null +++ b/bundles/aliyun/src/test/java/org/apache/gravitino/oss/credential/TestCredentialProvider.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.oss.credential; + +import org.apache.gravitino.credential.OSSSecretKeyCredential; +import org.apache.gravitino.credential.OSSTokenCredential; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCredentialProvider { + + @Test + void testCredentialProviderType() { + OSSTokenProvider ossTokenProvider = new OSSTokenProvider(); + Assertions.assertEquals( + OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE, ossTokenProvider.credentialType()); + + OSSSecretKeyProvider ossSecretKeyProvider = new OSSSecretKeyProvider(); + Assertions.assertEquals( + OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE, + ossSecretKeyProvider.credentialType()); + } +} diff --git a/bundles/aws/build.gradle.kts b/bundles/aws/build.gradle.kts index 45fda5485d5..da06c4d2cce 100644 --- a/bundles/aws/build.gradle.kts +++ b/bundles/aws/build.gradle.kts @@ -46,6 +46,12 @@ dependencies { implementation(libs.commons.lang3) implementation(libs.hadoop3.aws) implementation(libs.guava) + + testImplementation(project(":api")) + testImplementation(project(":core")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testRuntimeOnly(libs.junit.jupiter.engine) } tasks.withType(ShadowJar::class.java) { diff --git a/bundles/aws/src/test/java/org/apache/gravitino/s3/credential/TestCredentialProvider.java b/bundles/aws/src/test/java/org/apache/gravitino/s3/credential/TestCredentialProvider.java new file mode 100644 index 00000000000..c0aaeaefcf8 --- /dev/null +++ b/bundles/aws/src/test/java/org/apache/gravitino/s3/credential/TestCredentialProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.s3.credential; + +import org.apache.gravitino.credential.S3SecretKeyCredential; +import org.apache.gravitino.credential.S3TokenCredential; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCredentialProvider { + + @Test + void testCredentialProviderType() { + S3TokenProvider s3TokenProvider = new S3TokenProvider(); + Assertions.assertEquals( + S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE, s3TokenProvider.credentialType()); + + S3SecretKeyProvider s3SecretKeyProvider = new S3SecretKeyProvider(); + Assertions.assertEquals( + S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE, s3SecretKeyProvider.credentialType()); + } +} diff --git a/bundles/azure/build.gradle.kts b/bundles/azure/build.gradle.kts index 59d8cf5f806..8dbd6ed489e 100644 --- a/bundles/azure/build.gradle.kts +++ b/bundles/azure/build.gradle.kts @@ -47,6 +47,12 @@ dependencies { // runtime used implementation(libs.commons.logging) implementation(libs.guava) + + testImplementation(project(":api")) + testImplementation(project(":core")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testRuntimeOnly(libs.junit.jupiter.engine) } tasks.withType(ShadowJar::class.java) { diff --git a/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java index c2b684acbde..3ec9d56c284 100644 --- a/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java +++ b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java @@ -34,7 +34,6 @@ import java.util.Set; import org.apache.gravitino.credential.ADLSTokenCredential; import org.apache.gravitino.credential.Credential; -import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.CredentialContext; import org.apache.gravitino.credential.CredentialProvider; import org.apache.gravitino.credential.PathBasedCredentialContext; @@ -66,7 +65,7 @@ public void close() {} @Override public String credentialType() { - return CredentialConstants.ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE; + return ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE; } @Override diff --git a/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java index 726c4f2d996..c17a7cbc106 100644 --- a/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java +++ b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java @@ -22,7 +22,6 @@ import java.util.Map; import org.apache.gravitino.credential.AzureAccountKeyCredential; import org.apache.gravitino.credential.Credential; -import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.CredentialContext; import org.apache.gravitino.credential.CredentialProvider; import org.apache.gravitino.credential.config.AzureCredentialConfig; @@ -44,7 +43,7 @@ public void close() {} @Override public String credentialType() { - return CredentialConstants.AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE; + return AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE; } @Override diff --git a/bundles/azure/src/test/java/org/apache/gravitino/abs/credential/TestCredentialProvider.java b/bundles/azure/src/test/java/org/apache/gravitino/abs/credential/TestCredentialProvider.java new file mode 100644 index 00000000000..2a48d521a2a --- /dev/null +++ b/bundles/azure/src/test/java/org/apache/gravitino/abs/credential/TestCredentialProvider.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.abs.credential; + +import org.apache.gravitino.credential.ADLSTokenCredential; +import org.apache.gravitino.credential.AzureAccountKeyCredential; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCredentialProvider { + + @Test + void testCredentialProviderType() { + ADLSTokenProvider adlsTokenProvider = new ADLSTokenProvider(); + Assertions.assertEquals( + ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE, adlsTokenProvider.credentialType()); + + AzureAccountKeyProvider azureAccountKeyProvider = new AzureAccountKeyProvider(); + Assertions.assertEquals( + AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE, + azureAccountKeyProvider.credentialType()); + } +} diff --git a/bundles/gcp/build.gradle.kts b/bundles/gcp/build.gradle.kts index 6f21dc3d5af..95907f8a3bd 100644 --- a/bundles/gcp/build.gradle.kts +++ b/bundles/gcp/build.gradle.kts @@ -44,6 +44,12 @@ dependencies { implementation(libs.commons.logging) implementation(libs.google.auth.credentials) implementation(libs.google.auth.http) + + testImplementation(project(":api")) + testImplementation(project(":core")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testRuntimeOnly(libs.junit.jupiter.engine) } tasks.withType(ShadowJar::class.java) { diff --git a/bundles/gcp/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java index 94234b2d98e..3f7d5bcfaa3 100644 --- a/bundles/gcp/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java +++ b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java @@ -38,7 +38,6 @@ import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.credential.Credential; -import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.CredentialContext; import org.apache.gravitino.credential.CredentialProvider; import org.apache.gravitino.credential.GCSTokenCredential; @@ -68,7 +67,7 @@ public void close() {} @Override public String credentialType() { - return CredentialConstants.GCS_TOKEN_CREDENTIAL_PROVIDER_TYPE; + return GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE; } @Override diff --git a/bundles/gcp/src/test/java/org/apache/gravitino/gcs/credential/TestCredentialProvider.java b/bundles/gcp/src/test/java/org/apache/gravitino/gcs/credential/TestCredentialProvider.java new file mode 100644 index 00000000000..162e02c9872 --- /dev/null +++ b/bundles/gcp/src/test/java/org/apache/gravitino/gcs/credential/TestCredentialProvider.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.gcs.credential; + +import org.apache.gravitino.credential.GCSTokenCredential; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCredentialProvider { + + @Test + void testCredentialProviderType() { + GCSTokenProvider gcsTokenProvider = new GCSTokenProvider(); + Assertions.assertEquals( + GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE, gcsTokenProvider.credentialType()); + } +} diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index c766a86c141..7d552deb6bf 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -24,18 +24,9 @@ public class CredentialConstants { public static final String CREDENTIAL_PROVIDERS = "credential-providers"; public static final String CREDENTIAL_CACHE_EXPIRE_RATIO = "credential-cache-expire-ratio"; public static final String CREDENTIAL_CACHE_MAX_SIZE = "credential-cache-max-size"; - public static final String S3_TOKEN_CREDENTIAL_PROVIDER = "s3-token"; public static final String S3_TOKEN_EXPIRE_IN_SECS = "s3-token-expire-in-secs"; - - public static final String GCS_TOKEN_CREDENTIAL_PROVIDER_TYPE = "gcs-token"; - - public static final String OSS_TOKEN_CREDENTIAL_PROVIDER = "oss-token"; public static final String OSS_TOKEN_EXPIRE_IN_SECS = "oss-token-expire-in-secs"; - - public static final String ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE = "adls-token"; public static final String ADLS_TOKEN_EXPIRE_IN_SECS = "adls-token-expire-in-secs"; - public static final String AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE = "azure-account-key"; - private CredentialConstants() {} } diff --git a/clients/client-python/gravitino/api/credential/adls_token_credential.py b/clients/client-python/gravitino/api/credential/adls_token_credential.py index 40ad0eebbd9..641a69bae9a 100644 --- a/clients/client-python/gravitino/api/credential/adls_token_credential.py +++ b/clients/client-python/gravitino/api/credential/adls_token_credential.py @@ -25,7 +25,7 @@ class ADLSTokenCredential(Credential, ABC): """Represents ADLS token credential.""" - ADLS_SAS_TOKEN_CREDENTIAL_TYPE: str = "adls-sas-token" + ADLS_TOKEN_CREDENTIAL_TYPE: str = "adls-token" ADLS_DOMAIN: str = "dfs.core.windows.net" _STORAGE_ACCOUNT_NAME: str = "azure-storage-account-name" _SAS_TOKEN: str = "adls-sas-token" @@ -51,7 +51,7 @@ def credential_type(self) -> str: Returns: the type of the credential. """ - return self.ADLS_SAS_TOKEN_CREDENTIAL_TYPE + return self.ADLS_TOKEN_CREDENTIAL_TYPE def expire_time_in_ms(self) -> int: """Returns the expiration time of the credential in milliseconds since diff --git a/clients/client-python/gravitino/utils/credential_factory.py b/clients/client-python/gravitino/utils/credential_factory.py index 32d7465b806..5d566e509bc 100644 --- a/clients/client-python/gravitino/utils/credential_factory.py +++ b/clients/client-python/gravitino/utils/credential_factory.py @@ -46,7 +46,7 @@ def create( credential = OSSTokenCredential(credential_info, expire_time_in_ms) elif credential_type == OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE: credential = OSSSecretKeyCredential(credential_info, expire_time_in_ms) - elif credential_type == ADLSTokenCredential.ADLS_SAS_TOKEN_CREDENTIAL_TYPE: + elif credential_type == ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE: credential = ADLSTokenCredential(credential_info, expire_time_in_ms) elif ( credential_type diff --git a/clients/client-python/tests/unittests/test_credential_factory.py b/clients/client-python/tests/unittests/test_credential_factory.py index 4c4a91495a1..fddbbc098ba 100644 --- a/clients/client-python/tests/unittests/test_credential_factory.py +++ b/clients/client-python/tests/unittests/test_credential_factory.py @@ -156,7 +156,7 @@ def test_adls_token_credential(self): adls_credential.credential_type(), credential_info, expire_time ) self.assertEqual( - ADLSTokenCredential.ADLS_SAS_TOKEN_CREDENTIAL_TYPE, + ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE, check_credential.credential_type(), ) diff --git a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java index 6291b8857d7..7bb766a4628 100644 --- a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java +++ b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java @@ -153,11 +153,9 @@ void testADLSTokenCredential() { long expireTime = 100; Credential credential = CredentialFactory.create( - ADLSTokenCredential.ADLS_SAS_TOKEN_CREDENTIAL_TYPE, - adlsTokenCredentialInfo, - expireTime); + ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE, adlsTokenCredentialInfo, expireTime); Assertions.assertEquals( - ADLSTokenCredential.ADLS_SAS_TOKEN_CREDENTIAL_TYPE, credential.credentialType()); + ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE, credential.credentialType()); Assertions.assertInstanceOf(ADLSTokenCredential.class, credential); ADLSTokenCredential adlsTokenCredential = (ADLSTokenCredential) credential; diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java index b16d504e1ea..b663251e0e6 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.gravitino.abs.credential.ADLSLocationUtils; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.ADLSTokenCredential; import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.integration.test.util.BaseIT; @@ -92,7 +93,7 @@ private Map getADLSConfig() { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, - CredentialConstants.ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE); + ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, storageAccountName); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java index 42709162aaa..695b72ed4b3 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.AzureAccountKeyCredential; import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.integration.test.util.BaseIT; @@ -82,7 +83,7 @@ private Map getADLSConfig() { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, - CredentialConstants.AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE); + AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, storageAccountName); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java index 8f7821cb48a..11ee27bf449 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.GCSTokenCredential; import org.apache.gravitino.credential.config.GCSCredentialConfig; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.integration.test.util.BaseIT; @@ -73,7 +74,7 @@ private Map getGCSConfig() { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, - CredentialConstants.GCS_TOKEN_CREDENTIAL_PROVIDER_TYPE); + GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + GCSCredentialConfig.GRAVITINO_GCS_CREDENTIAL_FILE_PATH, diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java index f3aaafabb86..af70253d84f 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.OSSTokenCredential; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.DownloaderUtils; @@ -86,7 +87,7 @@ private Map getOSSConfig() { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, - CredentialConstants.OSS_TOKEN_CREDENTIAL_PROVIDER); + OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE); configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_REGION, region); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_ENDPOINT, endpoint); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java index d31278051ac..7e16273245d 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java @@ -26,6 +26,7 @@ import java.util.Map; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.S3TokenCredential; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.DownloaderUtils; @@ -87,7 +88,7 @@ private Map getS3Config() { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, - CredentialConstants.S3_TOKEN_CREDENTIAL_PROVIDER); + S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE); configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + S3Properties.GRAVITINO_S3_REGION, region); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKey); From 69e71a34180e9220aef819e92e09dcecf8075e46 Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Sat, 28 Dec 2024 22:11:05 +0800 Subject: [PATCH 43/47] [#5985] improvement(CLI): Fix role command that supports handling multiple values (#5988) ### What changes were proposed in this pull request? In `GravitinoOptions`, the role option supports multiple values, but the handleRoleCommand implementation currently only processes a single role value. CLI should support create and delete multiple roles simultaneously. ### Why are the changes needed? Fix: #5985 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? local test + UT ```bash gcli role create -m demo_metalake --role role1 role2 role3 # role1, role2, role3 created gcli role delete -m demo_metalake --role role1 role2 role3 # role1, role2, role3 deleted. gcli role delete -m demo_metalake --role role1 role2 role3 unknown # role1, role2, role3 deleted, but unknown is not deleted. gcli role details -m demo_metalake # Missing --role option. gcli role details -m demo_metalake --role roleA roleB # Exception in thread "main" java.lang.IllegalArgumentException: details requires only one role, but multiple are currently passed. ``` --- .../apache/gravitino/cli/ErrorMessages.java | 1 + .../gravitino/cli/GravitinoCommandLine.java | 33 ++++- .../gravitino/cli/TestableCommandLine.java | 8 +- .../gravitino/cli/commands/CreateRole.java | 15 +- .../gravitino/cli/commands/DeleteRole.java | 32 +++-- .../gravitino/cli/TestRoleCommands.java | 133 ++++++++++++++++-- 6 files changed, 184 insertions(+), 38 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java index 1d6db1a5acd..a5253664501 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java @@ -32,6 +32,7 @@ public class ErrorMessages { public static final String MISSING_NAME = "Missing --name option."; public static final String MISSING_GROUP = "Missing --group option."; public static final String MISSING_USER = "Missing --user option."; + public static final String MISSING_ROLE = "Missing --role option."; public static final String MISSING_TAG = "Missing --tag option."; public static final String METALAKE_EXISTS = "Metalake already exists."; public static final String CATALOG_EXISTS = "Catalog already exists."; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index bd9cc1f8551..8db838b32af 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -742,17 +742,26 @@ protected void handleRoleCommand() { String userName = line.getOptionValue(GravitinoOptions.LOGIN); FullName name = new FullName(line); String metalake = name.getMetalakeName(); - String role = line.getOptionValue(GravitinoOptions.ROLE); String[] privileges = line.getOptionValues(GravitinoOptions.PRIVILEGE); Command.setAuthenticationMode(auth, userName); + String[] roles = line.getOptionValues(GravitinoOptions.ROLE); + if (roles == null && !CommandActions.LIST.equals(command)) { + System.err.println(ErrorMessages.MISSING_ROLE); + Main.exit(-1); + } + + if (roles != null) { + roles = Arrays.stream(roles).distinct().toArray(String[]::new); + } + switch (command) { case CommandActions.DETAILS: if (line.hasOption(GravitinoOptions.AUDIT)) { - newRoleAudit(url, ignore, metalake, role).handle(); + newRoleAudit(url, ignore, metalake, getOneRole(roles, CommandActions.DETAILS)).handle(); } else { - newRoleDetails(url, ignore, metalake, role).handle(); + newRoleDetails(url, ignore, metalake, getOneRole(roles, CommandActions.DETAILS)).handle(); } break; @@ -761,20 +770,24 @@ protected void handleRoleCommand() { break; case CommandActions.CREATE: - newCreateRole(url, ignore, metalake, role).handle(); + newCreateRole(url, ignore, metalake, roles).handle(); break; case CommandActions.DELETE: boolean forceDelete = line.hasOption(GravitinoOptions.FORCE); - newDeleteRole(url, ignore, forceDelete, metalake, role).handle(); + newDeleteRole(url, ignore, forceDelete, metalake, roles).handle(); break; case CommandActions.GRANT: - newGrantPrivilegesToRole(url, ignore, metalake, role, name, privileges).handle(); + newGrantPrivilegesToRole( + url, ignore, metalake, getOneRole(roles, CommandActions.GRANT), name, privileges) + .handle(); break; case CommandActions.REVOKE: - newRevokePrivilegesFromRole(url, ignore, metalake, role, name, privileges).handle(); + newRevokePrivilegesFromRole( + url, ignore, metalake, getOneRole(roles, CommandActions.REMOVE), name, privileges) + .handle(); break; default: @@ -784,6 +797,12 @@ protected void handleRoleCommand() { } } + private String getOneRole(String[] roles, String command) { + Preconditions.checkArgument( + roles.length == 1, command + " requires only one role, but multiple are currently passed."); + return roles[0]; + } + /** * Handles the command execution for Columns based on command type and the command line options. */ diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java index effe0da1f10..f07244c0053 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java @@ -468,13 +468,13 @@ protected RoleAudit newRoleAudit(String url, boolean ignore, String metalake, St return new RoleAudit(url, ignore, metalake, role); } - protected CreateRole newCreateRole(String url, boolean ignore, String metalake, String role) { - return new CreateRole(url, ignore, metalake, role); + protected CreateRole newCreateRole(String url, boolean ignore, String metalake, String[] roles) { + return new CreateRole(url, ignore, metalake, roles); } protected DeleteRole newDeleteRole( - String url, boolean ignore, boolean force, String metalake, String role) { - return new DeleteRole(url, ignore, force, metalake, role); + String url, boolean ignore, boolean force, String metalake, String[] roles) { + return new DeleteRole(url, ignore, force, metalake, roles); } protected TagDetails newTagDetails(String url, boolean ignore, String metalake, String tag) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateRole.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateRole.java index fea6fe4a720..e821013471f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateRole.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/CreateRole.java @@ -19,6 +19,7 @@ package org.apache.gravitino.cli.commands; +import com.google.common.base.Joiner; import java.util.Collections; import org.apache.gravitino.cli.ErrorMessages; import org.apache.gravitino.client.GravitinoClient; @@ -27,7 +28,7 @@ public class CreateRole extends Command { protected String metalake; - protected String role; + protected String[] roles; /** * Create a new role. @@ -35,12 +36,12 @@ public class CreateRole extends Command { * @param url The URL of the Gravitino server. * @param ignoreVersions If true don't check the client/server versions match. * @param metalake The name of the metalake. - * @param role The name of the role. + * @param roles The array of roles. */ - public CreateRole(String url, boolean ignoreVersions, String metalake, String role) { + public CreateRole(String url, boolean ignoreVersions, String metalake, String[] roles) { super(url, ignoreVersions); this.metalake = metalake; - this.role = role; + this.roles = roles; } /** Create a new role. */ @@ -48,7 +49,9 @@ public CreateRole(String url, boolean ignoreVersions, String metalake, String ro public void handle() { try { GravitinoClient client = buildClient(metalake); - client.createRole(role, null, Collections.EMPTY_LIST); + for (String role : roles) { + client.createRole(role, null, Collections.EMPTY_LIST); + } } catch (NoSuchMetalakeException err) { exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (RoleAlreadyExistsException err) { @@ -57,6 +60,6 @@ public void handle() { exitWithError(exp.getMessage()); } - System.out.println(role + " created"); + System.out.println(Joiner.on(", ").join(roles) + " created"); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteRole.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteRole.java index f175d95043f..fa7c8cacc2c 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteRole.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/DeleteRole.java @@ -19,6 +19,9 @@ package org.apache.gravitino.cli.commands; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import java.util.List; import org.apache.gravitino.cli.AreYouSure; import org.apache.gravitino.cli.ErrorMessages; import org.apache.gravitino.client.GravitinoClient; @@ -26,9 +29,9 @@ import org.apache.gravitino.exceptions.NoSuchRoleException; public class DeleteRole extends Command { - + public static final Joiner COMMA_JOINER = Joiner.on(", ").skipNulls(); protected String metalake; - protected String role; + protected String[] roles; protected boolean force; /** @@ -38,28 +41,30 @@ public class DeleteRole extends Command { * @param ignoreVersions If true don't check the client/server versions match. * @param force Force operation. * @param metalake The name of the metalake. - * @param role The name of the role. + * @param roles The name of the role. */ public DeleteRole( - String url, boolean ignoreVersions, boolean force, String metalake, String role) { + String url, boolean ignoreVersions, boolean force, String metalake, String[] roles) { super(url, ignoreVersions); this.metalake = metalake; this.force = force; - this.role = role; + this.roles = roles; } /** Delete a role. */ @Override public void handle() { - boolean deleted = false; - if (!AreYouSure.really(force)) { return; } + List failedRoles = Lists.newArrayList(); + List successRoles = Lists.newArrayList(); try { GravitinoClient client = buildClient(metalake); - deleted = client.deleteRole(role); + for (String role : roles) { + (client.deleteRole(role) ? successRoles : failedRoles).add(role); + } } catch (NoSuchMetalakeException err) { exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (NoSuchRoleException err) { @@ -68,10 +73,15 @@ public void handle() { exitWithError(exp.getMessage()); } - if (deleted) { - System.out.println(role + " deleted."); + if (failedRoles.isEmpty()) { + System.out.println(COMMA_JOINER.join(successRoles) + " deleted."); } else { - System.out.println(role + " not deleted."); + System.err.println( + COMMA_JOINER.join(successRoles) + + " deleted, " + + "but " + + COMMA_JOINER.join(failedRoles) + + " is not deleted."); } } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java index 88b380d63ee..0e671067e3f 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java @@ -19,14 +19,20 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateRole; @@ -36,17 +42,30 @@ import org.apache.gravitino.cli.commands.RevokePrivilegesFromRole; import org.apache.gravitino.cli.commands.RoleAudit; import org.apache.gravitino.cli.commands.RoleDetails; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestRoleCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -71,7 +90,7 @@ void testRoleDetailsCommand() { when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); - when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("admin"); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)).thenReturn(new String[] {"admin"}); GravitinoCommandLine commandLine = spy( new GravitinoCommandLine( @@ -83,13 +102,31 @@ void testRoleDetailsCommand() { verify(mockDetails).handle(); } + @Test + void testRoleDetailsCommandWithMultipleRoles() { + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)) + .thenReturn(new String[] {"admin", "roleA", "roleB"}); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.DETAILS)); + + assertThrows(IllegalArgumentException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newRoleDetails( + eq(GravitinoCommandLine.DEFAULT_URL), eq(false), eq("metalake_demo"), any()); + } + @Test void testRoleAuditCommand() { RoleAudit mockAudit = mock(RoleAudit.class); when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); - when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("group"); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)).thenReturn(new String[] {"group"}); when(mockCommandLine.hasOption(GravitinoOptions.AUDIT)).thenReturn(true); GravitinoCommandLine commandLine = spy( @@ -108,14 +145,39 @@ void testCreateRoleCommand() { when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); - when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("admin"); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)).thenReturn(new String[] {"admin"}); GravitinoCommandLine commandLine = spy( new GravitinoCommandLine( mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.CREATE)); doReturn(mockCreate) .when(commandLine) - .newCreateRole(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "admin"); + .newCreateRole( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", new String[] {"admin"}); + commandLine.handleCommandLine(); + verify(mockCreate).handle(); + } + + @Test + void testCreateRolesCommand() { + CreateRole mockCreate = mock(CreateRole.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)) + .thenReturn(new String[] {"admin", "engineer", "scientist"}); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.CREATE)); + + doReturn(mockCreate) + .when(commandLine) + .newCreateRole( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq(new String[] {"admin", "engineer", "scientist"})); commandLine.handleCommandLine(); verify(mockCreate).handle(); } @@ -126,14 +188,45 @@ void testDeleteRoleCommand() { when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); - when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("admin"); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)).thenReturn(new String[] {"admin"}); GravitinoCommandLine commandLine = spy( new GravitinoCommandLine( mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.DELETE)); doReturn(mockDelete) .when(commandLine) - .newDeleteRole(GravitinoCommandLine.DEFAULT_URL, false, false, "metalake_demo", "admin"); + .newDeleteRole( + GravitinoCommandLine.DEFAULT_URL, + false, + false, + "metalake_demo", + new String[] {"admin"}); + commandLine.handleCommandLine(); + verify(mockDelete).handle(); + } + + @Test + void testDeleteRolesCommand() { + DeleteRole mockDelete = mock(DeleteRole.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)) + .thenReturn(new String[] {"admin", "engineer", "scientist"}); + when(mockCommandLine.hasOption(GravitinoOptions.FORCE)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.DELETE)); + + doReturn(mockDelete) + .when(commandLine) + .newDeleteRole( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq(false), + eq("metalake_demo"), + eq(new String[] {"admin", "engineer", "scientist"})); commandLine.handleCommandLine(); verify(mockDelete).handle(); } @@ -144,7 +237,7 @@ void testDeleteRoleForceCommand() { when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); - when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("admin"); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)).thenReturn(new String[] {"admin"}); when(mockCommandLine.hasOption(GravitinoOptions.FORCE)).thenReturn(true); GravitinoCommandLine commandLine = spy( @@ -152,7 +245,8 @@ void testDeleteRoleForceCommand() { mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.DELETE)); doReturn(mockDelete) .when(commandLine) - .newDeleteRole(GravitinoCommandLine.DEFAULT_URL, false, true, "metalake_demo", "admin"); + .newDeleteRole( + GravitinoCommandLine.DEFAULT_URL, false, true, "metalake_demo", new String[] {"admin"}); commandLine.handleCommandLine(); verify(mockDelete).handle(); } @@ -166,7 +260,7 @@ void testGrantPrivilegesToRole() { when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); - when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("admin"); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)).thenReturn(new String[] {"admin"}); when(mockCommandLine.hasOption(GravitinoOptions.PRIVILEGE)).thenReturn(true); when(mockCommandLine.getOptionValues(GravitinoOptions.PRIVILEGE)).thenReturn(privileges); GravitinoCommandLine commandLine = @@ -195,7 +289,7 @@ void testRevokePrivilegesFromRole() { when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(true); - when(mockCommandLine.getOptionValue(GravitinoOptions.ROLE)).thenReturn("admin"); + when(mockCommandLine.getOptionValues(GravitinoOptions.ROLE)).thenReturn(new String[] {"admin"}); when(mockCommandLine.hasOption(GravitinoOptions.PRIVILEGE)).thenReturn(true); when(mockCommandLine.getOptionValues(GravitinoOptions.PRIVILEGE)).thenReturn(privileges); GravitinoCommandLine commandLine = @@ -214,4 +308,23 @@ void testRevokePrivilegesFromRole() { commandLine.handleCommandLine(); verify(mockRevoke).handle(); } + + @Test + void testDeleteRoleCommandWithoutRole() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.ROLE)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.ROLE, CommandActions.REVOKE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteRole( + eq(GravitinoCommandLine.DEFAULT_URL), eq(false), eq(false), eq("metalake_demo"), any()); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_ROLE); + } } From 2e2033293d5851dda74f190d377eb3961f3204f6 Mon Sep 17 00:00:00 2001 From: Jimmy Lee <55496001+waukin@users.noreply.github.com> Date: Sat, 28 Dec 2024 22:12:32 +0800 Subject: [PATCH 44/47] [#6014] refactor: CLI output methods for no data hints (#6015) ### What changes were proposed in this pull request? In the `ListMetalakes` and `ListCatalogs` methods, retain the use of `output(metalakes)` and `output(catalogs)`. If metalakes or catalogs are empty arrays, they will be handled by the `output` method in `PlainFormat` and `TableFormat`. ### Why are the changes needed? Issue: https://github.com/apache/gravitino/issues/6014 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? No. --- .../gravitino/cli/commands/ListCatalogs.java | 6 +--- .../gravitino/cli/commands/ListMetalakes.java | 6 +--- .../gravitino/cli/outputs/PlainFormat.java | 24 +++++++++----- .../gravitino/cli/outputs/TableFormat.java | 32 ++++++++++++------- 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java index eb9c960b14e..e6aaf811ec9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListCatalogs.java @@ -49,11 +49,7 @@ public void handle() { try { GravitinoClient client = buildClient(metalake); catalogs = client.listCatalogsInfo(); - if (catalogs.length == 0) { - System.out.println("No catalogs exist."); - } else { - output(catalogs); - } + output(catalogs); } catch (NoSuchMetalakeException err) { exitWithError(ErrorMessages.UNKNOWN_METALAKE); } catch (Exception exp) { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java index b2388e5cd3d..ee5ac81d646 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListMetalakes.java @@ -43,11 +43,7 @@ public void handle() { try { GravitinoAdminClient client = buildAdminClient(); metalakes = client.listMetalakes(); - if (metalakes.length == 0) { - System.out.println("No metalakes exist."); - } else { - output(metalakes); - } + output(metalakes); } catch (Exception exp) { exitWithError(exp.getMessage()); } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/PlainFormat.java b/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/PlainFormat.java index 6160634db90..66e616c4f78 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/PlainFormat.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/PlainFormat.java @@ -50,10 +50,14 @@ public void output(Metalake metalake) { static final class MetalakesPlainFormat implements OutputFormat { @Override public void output(Metalake[] metalakes) { - List metalakeNames = - Arrays.stream(metalakes).map(Metalake::name).collect(Collectors.toList()); - String all = String.join(System.lineSeparator(), metalakeNames); - System.out.println(all); + if (metalakes.length == 0) { + System.out.println("No metalakes exist."); + } else { + List metalakeNames = + Arrays.stream(metalakes).map(Metalake::name).collect(Collectors.toList()); + String all = String.join(System.lineSeparator(), metalakeNames); + System.out.println(all); + } } } @@ -74,10 +78,14 @@ public void output(Catalog catalog) { static final class CatalogsPlainFormat implements OutputFormat { @Override public void output(Catalog[] catalogs) { - List catalogNames = - Arrays.stream(catalogs).map(Catalog::name).collect(Collectors.toList()); - String all = String.join(System.lineSeparator(), catalogNames); - System.out.println(all); + if (catalogs.length == 0) { + System.out.println("No catalogs exist."); + } else { + List catalogNames = + Arrays.stream(catalogs).map(Catalog::name).collect(Collectors.toList()); + String all = String.join(System.lineSeparator(), catalogNames); + System.out.println(all); + } } } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/TableFormat.java b/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/TableFormat.java index 6946ad13067..c9f502069b9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/TableFormat.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/TableFormat.java @@ -56,13 +56,17 @@ public void output(Metalake metalake) { static final class MetalakesTableFormat implements OutputFormat { @Override public void output(Metalake[] metalakes) { - List headers = Collections.singletonList("metalake"); - List> rows = new ArrayList<>(); - for (int i = 0; i < metalakes.length; i++) { - rows.add(Arrays.asList(metalakes[i].name())); + if (metalakes.length == 0) { + System.out.println("No metalakes exist."); + } else { + List headers = Collections.singletonList("metalake"); + List> rows = new ArrayList<>(); + for (int i = 0; i < metalakes.length; i++) { + rows.add(Arrays.asList(metalakes[i].name())); + } + TableFormatImpl tableFormat = new TableFormatImpl(); + tableFormat.print(headers, rows); } - TableFormatImpl tableFormat = new TableFormatImpl(); - tableFormat.print(headers, rows); } } @@ -85,13 +89,17 @@ public void output(Catalog catalog) { static final class CatalogsTableFormat implements OutputFormat { @Override public void output(Catalog[] catalogs) { - List headers = Collections.singletonList("catalog"); - List> rows = new ArrayList<>(); - for (int i = 0; i < catalogs.length; i++) { - rows.add(Arrays.asList(catalogs[i].name())); + if (catalogs.length == 0) { + System.out.println("No catalogs exist."); + } else { + List headers = Collections.singletonList("catalog"); + List> rows = new ArrayList<>(); + for (int i = 0; i < catalogs.length; i++) { + rows.add(Arrays.asList(catalogs[i].name())); + } + TableFormatImpl tableFormat = new TableFormatImpl(); + tableFormat.print(headers, rows); } - TableFormatImpl tableFormat = new TableFormatImpl(); - tableFormat.print(headers, rows); } } From cfe6691d9d0f47183b1ad92e68029a3b055de06a Mon Sep 17 00:00:00 2001 From: pancx Date: Sun, 29 Dec 2024 13:13:55 +0800 Subject: [PATCH 45/47] [#5831] fix(CLI): Fix CLi gives unexpected output when setting a tag 1. Change validation code; 2. add method to FullName; 3. fix some error of ErrorMessages; --- .../org/apache/gravitino/cli/FullName.java | 9 +++++++ .../gravitino/cli/GravitinoCommandLine.java | 27 +++++++------------ .../gravitino/cli/commands/UntagEntity.java | 24 ++++++++--------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java index a2be2e52c2d..a430bf02567 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java @@ -191,6 +191,15 @@ public boolean hasNamePart(int partNo) { return false; } + /** + * Are there any names that can be retrieved? + * + * @return True if the name exists, or false if it does not. + */ + public Boolean hasName() { + return line.hasOption(GravitinoOptions.NAME); + } + /** * Does the catalog name exist? * diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 8db838b32af..f8d6226f3c9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -667,14 +667,16 @@ protected void handleTagCommand() { if (propertySet != null && valueSet != null) { newSetTagProperty(url, ignore, metalake, getOneTag(tags), propertySet, valueSet).handle(); } else if (propertySet == null && valueSet == null) { - if (!hasEntity(name)) { - System.err.println(ErrorMessages.MALFORMED_NAME); - return; + if (!name.hasName()) { + System.err.println(ErrorMessages.MISSING_NAME); + Main.exit(-1); } newTagEntity(url, ignore, metalake, name, tags).handle(); } else { System.err.println( - "This command cannot be executed. The tag set command only supports configuring tag properties or attaching tags to entities."); + "Command cannot be executed. The set command only supports configuring tag properties or attaching " + + "tags to entity."); + Main.exit(-1); } break; @@ -682,19 +684,15 @@ protected void handleTagCommand() { boolean isTag = line.hasOption(GravitinoOptions.TAG); if (!isTag) { boolean forceRemove = line.hasOption(GravitinoOptions.FORCE); - if (!hasEntity(name)) { - System.err.println(ErrorMessages.MALFORMED_NAME); - return; - } newRemoveAllTags(url, ignore, metalake, name, forceRemove).handle(); } else { String propertyRemove = line.getOptionValue(GravitinoOptions.PROPERTY); if (propertyRemove != null) { newRemoveTagProperty(url, ignore, metalake, getOneTag(tags), propertyRemove).handle(); } else { - if (!hasEntity(name)) { - System.err.println(ErrorMessages.MALFORMED_NAME); - return; + if (!name.hasName()) { + System.err.println(ErrorMessages.MISSING_NAME); + Main.exit(-1); } newUntagEntity(url, ignore, metalake, name, tags).handle(); } @@ -728,13 +726,6 @@ private String getOneTag(String[] tags) { return tags[0]; } - private boolean hasEntity(FullName name) { - // TODO fileset and topic - return !(Objects.isNull(name.getCatalogName()) - && Objects.isNull(name.getSchemaName()) - && Objects.isNull(name.getTableName())); - } - /** Handles the command execution for Roles based on command type and the command line options. */ protected void handleRoleCommand() { String url = getUrl(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java index 36b806056cc..85afdfce118 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java @@ -19,6 +19,7 @@ package org.apache.gravitino.cli.commands; +import com.google.common.base.Joiner; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; @@ -32,6 +33,7 @@ import org.apache.gravitino.rel.Table; public class UntagEntity extends Command { + public static final Joiner COMMA_JOINER = Joiner.on(", ").skipNulls(); protected final String metalake; protected final FullName name; protected final String[] tags; @@ -91,24 +93,20 @@ public void handle() { } catch (NoSuchCatalogException err) { exitWithError(ErrorMessages.UNKNOWN_CATALOG); } catch (NoSuchSchemaException err) { - exitWithError(ErrorMessages.UNKNOWN_TABLE); + exitWithError(ErrorMessages.UNKNOWN_SCHEMA); } catch (NoSuchTableException err) { exitWithError(ErrorMessages.UNKNOWN_TABLE); } catch (Exception exp) { exitWithError(exp.getMessage()); } - String all = String.join(",", removeTags); - - if (all.equals("")) { - all = "nothing"; - } - - if (tags.length > 1) { - System.out.println( - entity + " removed tags " + String.join(",", tags) + " now tagged with " + all); - } else { - System.out.println(entity + " removed tag " + tags[0].toString() + " now tagged with " + all); - } + System.out.println( + entity + + " removed tag(s): [" + + COMMA_JOINER.join(tags) + + "], now tagged with " + + "[" + + COMMA_JOINER.join(removeTags) + + "]"); } } From 79ac31259aee4884e607faeb6f5da729177183b3 Mon Sep 17 00:00:00 2001 From: pancx Date: Sun, 29 Dec 2024 13:14:17 +0800 Subject: [PATCH 46/47] [#5831] fix(CLI): Fix CLi gives unexpected output when setting a tag Add some test case. --- .../apache/gravitino/cli/TestFulllName.java | 15 ++ .../apache/gravitino/cli/TestTagCommands.java | 144 ++++++++++++++++++ 2 files changed, 159 insertions(+) diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java index 4b5e1fed79b..e5ec92e1063 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java @@ -111,6 +111,21 @@ public void missingArgs() throws Exception { assertNull(namePart); } + @Test + public void hasPartName() throws ParseException { + String[] argsWithoutName = {"catalog", "details", "--metalake", "metalake"}; + CommandLine commandLineWithoutName = new DefaultParser().parse(options, argsWithoutName); + FullName fullNameWithoutName = new FullName(commandLineWithoutName); + assertFalse(fullNameWithoutName.hasName()); + + String[] argsWithName = { + "catalog", "details", "--metalake", "metalake", "--name", "Hive_catalog" + }; + CommandLine commandLineWithName = new DefaultParser().parse(options, argsWithName); + FullName fullNameWithName = new FullName(commandLineWithName); + assertTrue(fullNameWithName.hasName()); + } + @Test public void hasPartNameMetalake() throws Exception { String[] args = {"metalake", "details", "--metalake", "metalake"}; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java index 03c583e24f6..c7af9fb6abf 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; @@ -141,6 +142,30 @@ void testCreateTagCommand() { verify(mockCreate).handle(); } + @Test + void testCreateCommandWithoutTagOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.TAG)).thenReturn(false); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TAG, CommandActions.CREATE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newCreateTags( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + isNull(), + isNull()); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_TAG); + } + @Test void testCreateTagsCommand() { CreateTag mockCreate = mock(CreateTag.class); @@ -272,6 +297,68 @@ void testSetTagPropertyCommand() { verify(mockSetProperty).handle(); } + @Test + void testSetTagPropertyCommandWithoutPropertyOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.TAG)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.TAG)).thenReturn(new String[] {"tagA"}); + when(mockCommandLine.hasOption(GravitinoOptions.PROPERTY)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.VALUE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.VALUE)).thenReturn("value"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TAG, CommandActions.SET)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newSetTagProperty( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq("tagA"), + isNull(), + eq("value")); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + "Command cannot be executed. The set command only supports configuring tag properties or attaching " + + "tags to entity."); + } + + @Test + void testSetTagPropertyCommandWithoutValueOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.TAG)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.TAG)).thenReturn(new String[] {"tagA"}); + when(mockCommandLine.hasOption(GravitinoOptions.PROPERTY)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.PROPERTY)).thenReturn("property"); + when(mockCommandLine.hasOption(GravitinoOptions.VALUE)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TAG, CommandActions.SET)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newSetTagProperty( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq("tagA"), + eq("property"), + isNull()); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + "Command cannot be executed. The set command only supports configuring tag properties or attaching " + + "tags to entity."); + } + @Test void testSetMultipleTagPropertyCommandError() { when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); @@ -448,6 +535,32 @@ public boolean matches(String[] argument) { verify(mockTagEntity).handle(); } + @Test + void testTagEntityCommandWithoutName() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.TAG)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.TAG)).thenReturn(new String[] {"tagA"}); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TAG, CommandActions.SET)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTagEntity( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + isNull(), + argThat( + argument -> argument != null && argument.length > 0 && "tagA".equals(argument[0]))); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_NAME); + } + @Test void testTagsEntityCommand() { TagEntity mockTagEntity = mock(TagEntity.class); @@ -517,6 +630,37 @@ public boolean matches(String[] argument) { verify(mockUntagEntity).handle(); } + @Test + void testUntagEntityCommandWithoutName() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.TAG)).thenReturn(true); + when(mockCommandLine.getOptionValues(GravitinoOptions.TAG)) + .thenReturn(new String[] {"tagA", "tagB"}); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TAG, CommandActions.REMOVE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newUntagEntity( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + isNull(), + argThat( + argument -> + argument != null + && argument.length > 0 + && "tagA".equals(argument[0]) + && "tagB".equals(argument[1]))); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_NAME); + } + @Test void testUntagsEntityCommand() { UntagEntity mockUntagEntity = mock(UntagEntity.class); From d4ba7ccd4b3709edb7d664f000d0517ddfb14bcc Mon Sep 17 00:00:00 2001 From: pancx Date: Mon, 30 Dec 2024 10:24:43 +0800 Subject: [PATCH 47/47] [#5831] fix(CLI): Fix CLi gives unexpected output when setting a tag fix some error --- .../org/apache/gravitino/cli/FullName.java | 18 ++++++++--------- .../gravitino/cli/GravitinoCommandLine.java | 4 +--- .../gravitino/cli/commands/UntagEntity.java | 20 +++++++++++-------- .../apache/gravitino/cli/TestTagCommands.java | 10 ++-------- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java index a430bf02567..f2eef2a5a2d 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java @@ -148,6 +148,15 @@ public String getName() { return null; } + /** + * Are there any names that can be retrieved? + * + * @return True if the name exists, or false if it does not. + */ + public Boolean hasName() { + return line.hasOption(GravitinoOptions.NAME); + } + /** * Helper method to retrieve a specific part of the full name based on the position of the part. * @@ -191,15 +200,6 @@ public boolean hasNamePart(int partNo) { return false; } - /** - * Are there any names that can be retrieved? - * - * @return True if the name exists, or false if it does not. - */ - public Boolean hasName() { - return line.hasOption(GravitinoOptions.NAME); - } - /** * Does the catalog name exist? * diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index f8d6226f3c9..7869dd97b62 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -673,9 +673,7 @@ protected void handleTagCommand() { } newTagEntity(url, ignore, metalake, name, tags).handle(); } else { - System.err.println( - "Command cannot be executed. The set command only supports configuring tag properties or attaching " - + "tags to entity."); + System.err.println("The set command only supports tag properties or attaching tags."); Main.exit(-1); } break; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java index 85afdfce118..8f4a4a9cf02 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java @@ -100,13 +100,17 @@ public void handle() { exitWithError(exp.getMessage()); } - System.out.println( - entity - + " removed tag(s): [" - + COMMA_JOINER.join(tags) - + "], now tagged with " - + "[" - + COMMA_JOINER.join(removeTags) - + "]"); + String all = String.join(",", removeTags); + + if (all.equals("")) { + all = "nothing"; + } + + if (tags.length > 1) { + System.out.println( + entity + " removed tags " + String.join(",", tags) + " now tagged with " + all); + } else { + System.out.println(entity + " removed tag " + tags[0].toString() + " now tagged with " + all); + } } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java index c7af9fb6abf..74932ca87b3 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java @@ -322,10 +322,7 @@ void testSetTagPropertyCommandWithoutPropertyOption() { isNull(), eq("value")); String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); - assertEquals( - output, - "Command cannot be executed. The set command only supports configuring tag properties or attaching " - + "tags to entity."); + assertEquals(output, "The set command only supports tag properties or attaching tags."); } @Test @@ -353,10 +350,7 @@ void testSetTagPropertyCommandWithoutValueOption() { eq("property"), isNull()); String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); - assertEquals( - output, - "Command cannot be executed. The set command only supports configuring tag properties or attaching " - + "tags to entity."); + assertEquals(output, "The set command only supports tag properties or attaching tags."); } @Test