diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 959f0cf0d43bac..f86ee5dfbf4d9a 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -220,6 +220,8 @@ supportedCreateStatement | CREATE USER (IF NOT EXISTS)? grantUserIdentify (SUPERUSER | DEFAULT ROLE role=STRING_LITERAL)? passwordOption commentSpec? #createUser + | CREATE EXTERNAL? RESOURCE (IF NOT EXISTS)? + name=identifierOrText properties=propertyClause? #createResource ; supportedAlterStatement @@ -782,8 +784,6 @@ unsupportedCreateStatement : CREATE (DATABASE | SCHEMA) (IF NOT EXISTS)? name=multipartIdentifier properties=propertyClause? #createDatabase | CREATE (READ ONLY)? REPOSITORY name=identifier WITH storageBackend #createRepository - | CREATE EXTERNAL? RESOURCE (IF NOT EXISTS)? - name=identifierOrText properties=propertyClause? #createResource | CREATE STORAGE VAULT (IF NOT EXISTS)? name=identifierOrText properties=propertyClause? #createStorageVault | CREATE WORKLOAD POLICY (IF NOT EXISTS)? name=identifierOrText diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index 0f0fd7d5de6d4f..5c384d7783ba06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -26,6 +26,7 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -131,6 +132,14 @@ public static Resource fromStmt(CreateResourceStmt stmt) throws DdlException { return resource; } + public static Resource fromInfo(CreateResourceInfo info) throws DdlException { + Resource resource = getResourceInstance(info.getResourceType(), info.getResourceName()); + resource.id = Env.getCurrentEnv().getNextId(); + resource.version = 0; + resource.setProperties(info.getProperties()); + return resource; + } + public long getId() { return this.id; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java index b8580ffb796d24..8e4c1265b534b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java @@ -30,7 +30,9 @@ import org.apache.doris.common.proc.ProcNodeInterface; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand; import org.apache.doris.nereids.trees.plans.commands.DropResourceCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo; import org.apache.doris.persist.DropResourceOperationLog; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.policy.Policy; @@ -83,6 +85,18 @@ public void createResource(CreateResourceStmt stmt) throws DdlException { } } + public void createResource(CreateResourceCommand command) throws DdlException { + CreateResourceInfo info = command.getInfo(); + if (info.getResourceType() == ResourceType.UNKNOWN) { + throw new DdlException("Only support SPARK, ODBC_CATALOG ,JDBC, S3_COOLDOWN, S3, HDFS and HMS resource."); + } + Resource resource = Resource.fromInfo(info); + if (createResource(resource, info.isIfNotExists())) { + Env.getCurrentEnv().getEditLog().logCreateResource(resource); + LOG.info("Create resource success. Resource: {}", resource.getName()); + } + } + // Return true if the resource is truly added, // otherwise, return false or throw exception. public boolean createResource(Resource resource, boolean ifNotExists) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 67fa8eafdb6f5b..9d4e944ccc278a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -567,6 +567,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand; import org.apache.doris.nereids.trees.plans.commands.CreateRoleCommand; import org.apache.doris.nereids.trees.plans.commands.CreateSqlBlockRuleCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; @@ -737,6 +738,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp; import org.apache.doris.nereids.trees.plans.commands.info.CreateJobInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableLikeInfo; @@ -6621,5 +6623,20 @@ public UserDesc visitGrantUserIdentify(DorisParser.GrantUserIdentifyContext ctx) return new UserDesc(userIdentity, new PassVar(password, isPlain)); } + + @Override + public LogicalPlan visitCreateResource(DorisParser.CreateResourceContext ctx) { + String resourceName = visitIdentifierOrText(ctx.name); + Map properties = new HashMap<>(visitPropertyClause(ctx.properties)); + + CreateResourceInfo createResourceInfo = new CreateResourceInfo( + ctx.EXTERNAL() != null, + ctx.IF() != null, + resourceName, + properties + ); + + return new CreateResourceCommand(createResourceInfo); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 96db0af6fd49f9..44bc603d4700f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -340,5 +340,6 @@ public enum PlanType { TRANSACTION_ROLLBACK_COMMAND, KILL_ANALYZE_JOB_COMMAND, DROP_ANALYZE_JOB_COMMAND, - CREATE_USER_COMMAND + CREATE_USER_COMMAND, + CREATE_RESOURCE_COMMAND } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommand.java new file mode 100644 index 00000000000000..db7e989a0f3a03 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommand.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.StmtType; +import org.apache.doris.catalog.Env; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +/** + * create resource command + */ +public class CreateResourceCommand extends Command implements ForwardWithSync, NeedAuditEncryption { + private final CreateResourceInfo info; + + public CreateResourceCommand(CreateResourceInfo info) { + super(PlanType.CREATE_RESOURCE_COMMAND); + this.info = info; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCreateResourceCommand(this, context); + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + info.validate(); + Env.getCurrentEnv().getResourceMgr().createResource(this); + } + + @Override + public boolean needAuditEncryption() { + return true; + } + + @Override + public StmtType stmtType() { + return StmtType.CREATE; + } + + public CreateResourceInfo getInfo() { + return info; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java new file mode 100644 index 00000000000000..5ccba3e690422a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Resource.ResourceType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.constants.AzureProperties; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Strings; + +import java.util.Map; + +/** + * create resource info + */ +public class CreateResourceInfo { + private static final String TYPE = "type"; + private final boolean isExternal; + private final boolean ifNotExists; + private final String resourceName; + private final Map properties; + private ResourceType resourceType; + + /** + * CreateResourceInfo + */ + public CreateResourceInfo(boolean isExternal, boolean ifNotExists, String resourceName, + Map properties) { + this.isExternal = isExternal; + this.ifNotExists = ifNotExists; + this.resourceName = resourceName; + this.properties = properties; + this.resourceType = ResourceType.UNKNOWN; + } + + /** + * analyze createResourceInfo + */ + public void validate() throws UserException { + // check auth + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + + // check name + FeNameFormat.checkResourceName(resourceName, ResourceTypeEnum.GENERAL); + + // check type in properties + if (properties == null || properties.isEmpty()) { + throw new AnalysisException("Resource properties can't be null"); + } + + analyzeResourceType(); + } + + /** + * analyze Resource Type + */ + public void analyzeResourceType() throws UserException { + String type = null; + for (Map.Entry property : properties.entrySet()) { + if (property.getKey().equalsIgnoreCase(TYPE)) { + type = property.getValue(); + } + } + if (Strings.isNullOrEmpty(type)) { + throw new AnalysisException("Resource type can't be null"); + } + + if (AzureProperties.checkAzureProviderPropertyExist(properties)) { + resourceType = ResourceType.AZURE; + return; + } + + resourceType = ResourceType.fromString(type); + if (resourceType == ResourceType.UNKNOWN) { + throw new AnalysisException("Unsupported resource type: " + type); + } + if (resourceType == ResourceType.SPARK && !isExternal) { + throw new AnalysisException("Spark is external resource"); + } + if (resourceType == ResourceType.ODBC_CATALOG && !Config.enable_odbc_mysql_broker_table) { + throw new AnalysisException("ODBC table is deprecated, use JDBC instead. Or you can set " + + "`enable_odbc_mysql_broker_table=true` in fe.conf to enable ODBC again."); + } + } + + public boolean isExternal() { + return isExternal; + } + + public boolean isIfNotExists() { + return ifNotExists; + } + + public String getResourceName() { + return resourceName; + } + + public Map getProperties() { + return properties; + } + + public ResourceType getResourceType() { + return resourceType; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index c60ae16042992c..948c416f5c1728 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -56,6 +56,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand; import org.apache.doris.nereids.trees.plans.commands.CreateRoleCommand; import org.apache.doris.nereids.trees.plans.commands.CreateSqlBlockRuleCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; @@ -979,4 +980,8 @@ default R visitDropAnalyzeJobCommand(DropAnalyzeJobCommand dropAnalyzeJobCommand default R visitCreateUserCommand(CreateUserCommand createUserCommand, C context) { return visitCommand(createUserCommand, context); } + + default R visitCreateResourceCommand(CreateResourceCommand createResourceCommand, C context) { + return visitCommand(createResourceCommand, context); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommandTest.java new file mode 100644 index 00000000000000..6245c28b919f6b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommandTest.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableMap; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +public class CreateResourceCommandTest extends TestWithFeService { + @Test + public void testValidate(@Mocked Env env, @Mocked AccessControllerManager accessManager) { + new Expectations() { + { + env.getAccessManager(); + result = accessManager; + accessManager.checkGlobalPriv((ConnectContext) any, PrivPredicate.ADMIN); + result = true; + } + }; + + // test validate normal + Map properties = ImmutableMap.of("type", "spark", "host", "http://127.0.0.1:29200"); + CreateResourceInfo info = new CreateResourceInfo(true, false, "test", properties); + CreateResourceCommand createResourceCommand = new CreateResourceCommand(info); + Assertions.assertDoesNotThrow(() -> createResourceCommand.getInfo().validate()); + + // test validate abnormal + // test properties + info = new CreateResourceInfo(false, false, "test", null); + CreateResourceCommand createResourceCommand1 = new CreateResourceCommand(info); + Assertions.assertThrows(AnalysisException.class, () -> createResourceCommand1.getInfo().validate()); + + // test resource type + properties = ImmutableMap.of("host", "http://127.0.0.1:29200"); + info = new CreateResourceInfo(false, false, "test", properties); + CreateResourceCommand createResourceCommand2 = new CreateResourceCommand(info); + Assertions.assertThrows(AnalysisException.class, () -> createResourceCommand2.getInfo().validate()); + + // test unsupported resource type + properties = ImmutableMap.of("type", "flink", "host", "http://127.0.0.1:29200"); + info = new CreateResourceInfo(false, false, "test", properties); + CreateResourceCommand createResourceCommand3 = new CreateResourceCommand(info); + Assertions.assertThrows(AnalysisException.class, () -> createResourceCommand3.getInfo().validate()); + + // test spark is external resource + properties = ImmutableMap.of("type", "spark", "host", "http://127.0.0.1:29200"); + info = new CreateResourceInfo(false, false, "test", properties); + CreateResourceCommand createResourceCommand4 = new CreateResourceCommand(info); + Assertions.assertThrows(AnalysisException.class, () -> createResourceCommand4.getInfo().validate()); + } + + @Test + public void testCreateResource() { + String es = "CREATE RESOURCE \"es_resource\"\n" + + "PROPERTIES\n" + + "(\n" + + " \"type\"=\"es\",\n" + + " \"hosts\"=\"http://127.0.0.1:29200\",\n" + + " \"nodes_discovery\"=\"false\",\n" + + " \"enable_keyword_sniff\"=\"true\"\n" + + ");"; + + String jdbc = "CREATE EXTERNAL RESOURCE \"jdbc_resource\"\n" + + "PROPERTIES\n" + + "(\n" + + " \"type\" = \"jdbc\",\n" + + " \"user\" = \"jdbc_user\",\n" + + " \"password\" = \"jdbc_passwd\",\n" + + " \"jdbc_url\" = \"jdbc:mysql://127.0.0.1:3316/doris_test?useSSL=false\",\n" + + " \"driver_url\" = \"https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar\",\n" + + " \"driver_class\" = \"com.mysql.cj.jdbc.Driver\"\n" + + ");"; + + Assertions.assertDoesNotThrow(() -> createResource(es)); + Assertions.assertDoesNotThrow(() -> createResource(jdbc)); + } + + private void createResource(String sql) throws Exception { + LogicalPlan plan = new NereidsParser().parseSingle(sql); + Assertions.assertTrue(plan instanceof CreateResourceCommand); + ((CreateResourceCommand) plan).run(connectContext, null); + } +}