diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run20.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run20.sql new file mode 100644 index 00000000000000..dba9680f456186 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run20.sql @@ -0,0 +1,2 @@ +create database if not exists nested.db1; + diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java index a9679583698eff..3f81f4464db22a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java @@ -53,7 +53,7 @@ public void analyze(Analyzer analyzer) throws UserException { Util.checkCatalogAllRules(catalogName); if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { - throw new AnalysisException("Internal catalog can't be drop."); + throw new AnalysisException("Internal catalog can't be dropped."); } if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv( diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index aaf7649ba4fb7b..580922167973a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3350,12 +3350,12 @@ public void createDb(CreateDbStmt stmt) throws DdlException { if (StringUtils.isEmpty(stmt.getCtlName())) { catalogIf = getCurrentCatalog(); } else { - catalogIf = catalogMgr.getCatalog(stmt.getCtlName()); + catalogIf = catalogMgr.getCatalogOrDdlException(stmt.getCtlName()); } catalogIf.createDb(stmt.getFullDbName(), stmt.isSetIfNotExists(), stmt.getProperties()); } - // For replay edit log, need't lock metadata + // For replay edit log, no need to lock metadata public void unprotectCreateDb(Database db) { getInternalCatalog().unprotectCreateDb(db); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index c6c751a88c6761..c3422fb7f1c5be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -25,6 +25,7 @@ import org.apache.doris.mysql.privilege.Role; import org.apache.doris.mysql.privilege.RoleManager; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.VariableMgr; import com.google.common.base.Strings; @@ -53,6 +54,10 @@ public class FeNameFormat { public static final String TEMPORARY_TABLE_SIGN = "_#TEMP#_"; + private static final String NESTED_DB_NAME_REGEX = "^[a-zA-Z][a-zA-Z0-9\\-_]*(\\.([a-zA-Z0-9\\-_]+))*$"; + private static final String NESTED_UNICODE_DB_NAME_REGEX + = "^[a-zA-Z\\p{L}][a-zA-Z0-9\\-_\\p{L}]*(\\.([a-zA-Z0-9\\-_\\p{L}]+))*$"; + public static void checkCatalogName(String catalogName) throws AnalysisException { if (!InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalogName) && (Strings.isNullOrEmpty(catalogName) || !catalogName.matches(getCommonNameRegex()))) { @@ -61,7 +66,10 @@ public static void checkCatalogName(String catalogName) throws AnalysisException } public static void checkDbName(String dbName) throws AnalysisException { - if (Strings.isNullOrEmpty(dbName) || !dbName.matches(getCommonNameRegex())) { + if (Strings.isNullOrEmpty(dbName)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName); + } + if (!dbName.matches(getDbNameRegex())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName); } } @@ -253,6 +261,22 @@ public static String getCommonNameRegex() { } } + public static String getDbNameRegex() { + if (GlobalVariable.enableNestedNamespace) { + if (FeNameFormat.isEnableUnicodeNameSupport()) { + return NESTED_UNICODE_DB_NAME_REGEX; + } else { + return NESTED_DB_NAME_REGEX; + } + } else { + if (FeNameFormat.isEnableUnicodeNameSupport()) { + return UNICODE_COMMON_NAME_REGEX; + } else { + return COMMON_NAME_REGEX; + } + } + } + public static String getOutfileSuccessFileNameRegex() { if (FeNameFormat.isEnableUnicodeNameSupport()) { return UNICODE_UNDERSCORE_COMMON_NAME_REGEX; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 2b863b7e1138ed..1564c89ce5393c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -160,6 +160,12 @@ public CatalogIf getCatalogOrAnalysisException(long id) throws AnalysisException ErrorCode.ERR_UNKNOWN_CATALOG)); } + public CatalogIf getCatalogOrDdlException(long id) throws DdlException { + return getCatalogOrException(id, + catalog -> new DdlException(ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(catalog), + ErrorCode.ERR_UNKNOWN_CATALOG)); + } + public CatalogIf> getCatalogOrException(long id, Function e) throws E { CatalogIf catalog = idToCatalog.get(id); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 4818b722fcf436..0582a6cd8b451e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -34,6 +34,8 @@ import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.operations.ExternalMetadataOps; +import org.apache.doris.datasource.property.metastore.IcebergRestProperties; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions; import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; @@ -41,6 +43,8 @@ import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo; import org.apache.doris.nereids.trees.plans.commands.info.TagOptions; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.ManageSnapshots; import org.apache.iceberg.PartitionSpec; @@ -54,20 +58,24 @@ import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.view.View; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; +import java.util.stream.Stream; public class IcebergMetadataOps implements ExternalMetadataOps { @@ -129,15 +137,35 @@ public boolean databaseExist(String dbName) { public List listDatabaseNames() { try { - return executionAuthenticator.execute(() -> nsCatalog.listNamespaces(getNamespace()) - .stream() - .map(n -> n.level(n.length() - 1)) - .collect(Collectors.toList())); + return executionAuthenticator.execute(() -> listNestedNamespaces(getNamespace())); } catch (Exception e) { throw new RuntimeException("Failed to list database names, error message is:" + e.getMessage(), e); } } + @NotNull + private List listNestedNamespaces(Namespace parentNs) { + // Handle nested namespaces for Iceberg REST catalog, + // only if "iceberg.rest.nested-namespace-enabled" is true. + if (dorisCatalog instanceof IcebergRestExternalCatalog) { + IcebergRestExternalCatalog restCatalog = (IcebergRestExternalCatalog) dorisCatalog; + MetastoreProperties metaProps = restCatalog.getCatalogProperty().getMetastoreProperties(); + if (metaProps instanceof IcebergRestProperties + && ((IcebergRestProperties) metaProps).isIcebergRestNestedNamespaceEnabled()) { + return nsCatalog.listNamespaces(parentNs) + .stream() + .flatMap(childNs -> Stream.concat( + Stream.of(childNs.toString()), + listNestedNamespaces(childNs).stream() + )).collect(Collectors.toList()); + } + } + + return nsCatalog.listNamespaces(parentNs) + .stream() + .map(n -> n.level(n.length() - 1)) + .collect(Collectors.toList()); + } @Override public List listTableNames(String dbName) { @@ -210,7 +238,7 @@ private boolean performCreateDb(String dbName, boolean ifNotExists, Map { - preformDropDb(dbName, ifExists, force); + performDropDb(dbName, ifExists, force); return null; }); } catch (Exception e) { @@ -219,7 +247,7 @@ public void dropDbImpl(String dbName, boolean ifExists, boolean force) throws Dd } } - private void preformDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { + private void performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { ExternalDatabase dorisDb = dorisCatalog.getDbNullable(dbName); if (dorisDb == null) { if (ifExists) { @@ -230,21 +258,27 @@ private void preformDropDb(String dbName, boolean ifExists, boolean force) throw } } if (force) { - // try to drop all tables in the database - List remoteTableNames = listTableNames(dorisDb.getRemoteName()); - for (String remoteTableName : remoteTableNames) { - performDropTable(dorisDb.getRemoteName(), remoteTableName, true); - } - if (!remoteTableNames.isEmpty()) { - LOG.info("drop database[{}] with force, drop all tables, num: {}", dbName, remoteTableNames.size()); - } - // try to drop all views in the database - List remoteViewNames = listViewNames(dorisDb.getRemoteName()); - for (String remoteViewName : remoteViewNames) { - performDropView(dorisDb.getRemoteName(), remoteViewName); - } - if (!remoteViewNames.isEmpty()) { - LOG.info("drop database[{}] with force, drop all views, num: {}", dbName, remoteViewNames.size()); + try { + // try to drop all tables in the database + List remoteTableNames = listTableNames(dorisDb.getRemoteName()); + for (String remoteTableName : remoteTableNames) { + performDropTable(dorisDb.getRemoteName(), remoteTableName, true); + } + if (!remoteTableNames.isEmpty()) { + LOG.info("drop database[{}] with force, drop all tables, num: {}", dbName, remoteTableNames.size()); + } + // try to drop all views in the database + List remoteViewNames = listViewNames(dorisDb.getRemoteName()); + for (String remoteViewName : remoteViewNames) { + performDropView(dorisDb.getRemoteName(), remoteViewName); + } + if (!remoteViewNames.isEmpty()) { + LOG.info("drop database[{}] with force, drop all views, num: {}", dbName, remoteViewNames.size()); + } + } catch (NoSuchNamespaceException e) { + // just ignore + LOG.info("drop database[{}] force which does not exist", dbName); + return; } } nsCatalog.dropNamespace(getNamespace(dorisDb.getRemoteName())); @@ -761,7 +795,8 @@ public View loadView(String dbName, String tblName) { } try { ViewCatalog viewCatalog = (ViewCatalog) catalog; - return executionAuthenticator.execute(() -> viewCatalog.loadView(TableIdentifier.of(dbName, tblName))); + return executionAuthenticator.execute( + () -> viewCatalog.loadView(TableIdentifier.of(getNamespace(dbName), tblName))); } catch (Exception e) { throw new RuntimeException("Failed to load view, error message is:" + e.getMessage(), e); } @@ -774,7 +809,7 @@ public List listViewNames(String db) { } try { return executionAuthenticator.execute(() -> - ((ViewCatalog) catalog).listViews(Namespace.of(db)) + ((ViewCatalog) catalog).listViews(getNamespace(db)) .stream().map(TableIdentifier::name).collect(Collectors.toList())); } catch (Exception e) { throw new RuntimeException("Failed to list view names, error message is:" + e.getMessage(), e); @@ -782,15 +817,22 @@ public List listViewNames(String db) { } private TableIdentifier getTableIdentifier(String dbName, String tblName) { - return externalCatalogName - .map(s -> TableIdentifier.of(s, dbName, tblName)) - .orElseGet(() -> TableIdentifier.of(dbName, tblName)); + Namespace ns = getNamespace(dbName); + return TableIdentifier.of(ns, tblName); } private Namespace getNamespace(String dbName) { - return externalCatalogName - .map(s -> Namespace.of(s, dbName)) - .orElseGet(() -> Namespace.of(dbName)); + return getNamespace(externalCatalogName, dbName); + } + + @VisibleForTesting + public static Namespace getNamespace(Optional catalogName, String dbName) { + String[] splits = Splitter.on(".").omitEmptyStrings().trimResults().splitToList(dbName).toArray(new String[0]); + if (catalogName.isPresent()) { + splits = Arrays.copyOf(splits, splits.length + 1); + splits[splits.length - 1] = catalogName.get(); + } + return Namespace.of(splits); } private Namespace getNamespace() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java index fb64584c1cccb5..b899116a1a835e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java @@ -110,9 +110,8 @@ public class IcebergRestProperties extends AbstractIcebergProperties { @ConnectorProperty(names = {"iceberg.rest.nested-namespace-enabled"}, required = false, - supported = false, description = "Enable nested namespace for the iceberg rest catalog service.") - private String icebergRestNestedNamespaceEnabled = "true"; + private String icebergRestNestedNamespaceEnabled = "false"; @ConnectorProperty(names = {"iceberg.rest.case-insensitive-name-matching"}, required = false, @@ -201,11 +200,7 @@ private ParamRules buildRules() { ParamRules rules = new ParamRules() // OAuth2 requires either credential or token, but not both .mutuallyExclusive(icebergRestOauth2Credential, icebergRestOauth2Token, - "OAuth2 cannot have both credential and token configured") - // If using credential flow, server URI is required - .requireAllIfPresent(icebergRestOauth2Credential, - new String[] {icebergRestOauth2ServerUri}, - "OAuth2 credential flow requires server-uri"); + "OAuth2 cannot have both credential and token configured"); // Custom validation: OAuth2 scope should not be used with token if (Strings.isNotBlank(icebergRestOauth2Token) && Strings.isNotBlank(icebergRestOauth2Scope)) { @@ -275,7 +270,9 @@ private void addOAuth2Properties() { if (Strings.isNotBlank(icebergRestOauth2Credential)) { // Client Credentials Flow icebergRestCatalogProperties.put(OAuth2Properties.CREDENTIAL, icebergRestOauth2Credential); - icebergRestCatalogProperties.put(OAuth2Properties.OAUTH2_SERVER_URI, icebergRestOauth2ServerUri); + if (Strings.isNotBlank(icebergRestOauth2ServerUri)) { + icebergRestCatalogProperties.put(OAuth2Properties.OAUTH2_SERVER_URI, icebergRestOauth2ServerUri); + } if (Strings.isNotBlank(icebergRestOauth2Scope)) { icebergRestCatalogProperties.put(OAuth2Properties.SCOPE, icebergRestOauth2Scope); } @@ -306,6 +303,10 @@ public boolean isIcebergRestVendedCredentialsEnabled() { return Boolean.parseBoolean(icebergRestVendedCredentialsEnabled); } + public boolean isIcebergRestNestedNamespaceEnabled() { + return Boolean.parseBoolean(icebergRestNestedNamespaceEnabled); + } + /** * Unified method to configure FileIO properties for Iceberg catalog. * This method handles all storage types (HDFS, S3, MinIO, etc.) and populates diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java index 5c938ed6bc978d..58596669676047 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java @@ -18,14 +18,15 @@ package org.apache.doris.mysql; import org.apache.doris.catalog.Env; -import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.Pair; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ConnectContextUtil; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -33,6 +34,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Optional; // MySQL protocol util public class MysqlProto { @@ -225,42 +227,9 @@ public static boolean negotiate(ConnectContext context) throws IOException { // set database String db = authPacket.getDb(); if (!Strings.isNullOrEmpty(db)) { - String catalogName = null; - String dbName = null; - String[] dbNames = db.split("\\."); - if (dbNames.length == 1) { - dbName = db; - } else if (dbNames.length == 2) { - catalogName = dbNames[0]; - dbName = dbNames[1]; - } else if (dbNames.length > 2) { - context.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "Only one dot can be in the name: " + db); - sendResponsePacket(context); - return false; - } - - // mysql -d - if (Config.isCloudMode()) { - try { - dbName = ((CloudEnv) Env.getCurrentEnv()).analyzeCloudCluster(dbName, context); - } catch (DdlException e) { - context.getState().setError(e.getMysqlErrorCode(), e.getMessage()); - sendResponsePacket(context); - return false; - } - - if (dbName == null || dbName.isEmpty()) { - return true; - } - } - - try { - if (catalogName != null) { - context.getEnv().changeCatalog(context, catalogName); - } - Env.getCurrentEnv().changeDb(context, dbName); - } catch (DdlException e) { - context.getState().setError(e.getMysqlErrorCode(), e.getMessage()); + Optional> res = ConnectContextUtil.initCatalogAndDb(context, db); + if (res.isPresent()) { + context.getState().setError(res.get().first, res.get().second); sendResponsePacket(context); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java new file mode 100644 index 00000000000000..48c7744b570909 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.Util; +import org.apache.doris.nereids.StatementContext; + +import java.util.Optional; +import java.util.stream.Stream; + +public class ConnectContextUtil { + + // Sometimes it's necessary to parse SQL, but not in a user thread where no ConnectContext exists. + // In such cases, we need to simulate one—for example, + // when replaying metadata to parse materialized view (MV) creation statements. + // Note: After calling this method, ensure to invoke the cleanup() method of ConnectContext. + public static ConnectContext getDummyCtx(String dbName) { + ConnectContext ctx = new ConnectContext(); + ctx.setDatabase(dbName); + StatementContext statementContext = new StatementContext(); + statementContext.setConnectContext(ctx); + ctx.setStatementContext(statementContext); + ctx.setEnv(Env.getCurrentEnv()); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + ctx.getState().reset(); + ctx.setThreadLocalInfo(); + return ctx; + } + + public static Optional> initCatalogAndDb(ConnectContext ctx, String fullDbName) { + String catalogName = null; + String dbName = null; + String[] dbNames = fullDbName.split("\\."); + if (dbNames.length == 1) { + dbName = fullDbName; + } else if (dbNames.length == 2) { + catalogName = dbNames[0]; + dbName = dbNames[1]; + } else if (dbNames.length > 2) { + if (GlobalVariable.enableNestedNamespace) { + // use the first part as catalog name, the rest part as db name + catalogName = dbNames[0]; + dbName = Stream.of(dbNames).skip(1).reduce((a, b) -> a + "." + b).get(); + } else { + return Optional.of( + Pair.of(ErrorCode.ERR_BAD_DB_ERROR, "Only one dot can be in the name: " + fullDbName)); + } + } + + // mysql client + if (Config.isCloudMode()) { + try { + dbName = ((CloudEnv) ctx.getEnv()).analyzeCloudCluster(dbName, ctx); + } catch (DdlException e) { + return Optional.of(Pair.of(e.getMysqlErrorCode(), e.getMessage())); + } + if (dbName == null || dbName.isEmpty()) { + return Optional.empty(); + } + } + + try { + if (catalogName != null) { + ctx.getEnv().changeCatalog(ctx, catalogName); + } + ctx.getEnv().changeDb(ctx, dbName); + } catch (DdlException e) { + return Optional.of(Pair.of(e.getMysqlErrorCode(), e.getMessage())); + } catch (Throwable t) { + return Optional.of(Pair.of(ErrorCode.ERR_INTERNAL_ERROR, Util.getRootCauseMessage(t))); + } + ctx.getState().setOk(); + return Optional.empty(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 26262681b23721..8b7e3c8ac4d8f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -28,21 +28,18 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; -import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.qe.ComputeGroupException; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.ConnectionException; -import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.SqlUtils; -import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.metric.MetricRepo; @@ -121,45 +118,10 @@ public boolean isHandleQueryInFe() { // change current database of this session. protected void handleInitDb(String fullDbName) { - String catalogName = null; - String dbName = null; - String[] dbNames = fullDbName.split("\\."); - if (dbNames.length == 1) { - dbName = fullDbName; - } else if (dbNames.length == 2) { - catalogName = dbNames[0]; - dbName = dbNames[1]; - } else if (dbNames.length > 2) { - ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "Only one dot can be in the name: " + fullDbName); - return; - } - - // mysql client - if (Config.isCloudMode()) { - try { - dbName = ((CloudEnv) ctx.getEnv()).analyzeCloudCluster(dbName, ctx); - } catch (DdlException e) { - ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage()); - return; - } - if (dbName == null || dbName.isEmpty()) { - return; - } + Optional> res = ConnectContextUtil.initCatalogAndDb(ctx, fullDbName); + if (res.isPresent()) { + ctx.getState().setError(res.get().first, res.get().second); } - - try { - if (catalogName != null) { - ctx.getEnv().changeCatalog(ctx, catalogName); - } - ctx.getEnv().changeDb(ctx, dbName); - } catch (DdlException e) { - ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage()); - return; - } catch (Throwable t) { - ctx.getState().setError(ErrorCode.ERR_INTERNAL_ERROR, Util.getRootCauseMessage(t)); - return; - } - ctx.getState().setOk(); } // set killed flag diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index 12dd6cd24097bc..8bdf51a3b8d7c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -78,6 +78,8 @@ public final class GlobalVariable { public static final String ENABLE_FETCH_ICEBERG_STATS = "enable_fetch_iceberg_stats"; + public static final String ENABLE_NESTED_NAMESPACE = "enable_nested_namespace"; + @VariableMgr.VarAttr(name = VARIABLE_VERSION, flag = VariableMgr.INVISIBLE | VariableMgr.READ_ONLY | VariableMgr.GLOBAL) public static int variableVersion = CURRENT_VARIABLE_VERSION; @@ -203,6 +205,16 @@ public final class GlobalVariable { "Enable fetch stats for HMS Iceberg table when it's not analyzed."}) public static boolean enableFetchIcebergStats = false; + @VariableMgr.VarAttr(name = ENABLE_NESTED_NAMESPACE, flag = VariableMgr.GLOBAL, + description = { + "是否允许访问 `ns1.ns2` 这种类型的 database。当前仅适用于 External Catalog 中映射 Database 并访问。" + + "不支持创建。", + "Whether to allow accessing databases of the form `ns1.ns2`. " + + "Currently, this only applies to mapping databases in " + + "External Catalogs and accessing them. " + + "Creation is not supported."}) + public static boolean enableNestedNamespace = false; + // Don't allow creating instance. private GlobalVariable() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index be5e437da0a4e2..67bdaa3e390e47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -21,14 +21,11 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MysqlColType; -import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.common.AuthenticationException; -import org.apache.doris.common.Config; import org.apache.doris.common.ConnectionException; -import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.common.Pair; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlCommand; @@ -58,6 +55,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * Process one mysql connection, receive one packet, process, send one packet. @@ -349,47 +347,9 @@ private void handleChangeUser() throws IOException { if (Strings.isNullOrEmpty(db)) { ctx.changeDefaultCatalog(InternalCatalog.INTERNAL_CATALOG_NAME); } else { - String catalogName = null; - String dbName = null; - String[] dbNames = db.split("\\."); - if (dbNames.length == 1) { - dbName = db; - } else if (dbNames.length == 2) { - catalogName = dbNames[0]; - dbName = dbNames[1]; - } else if (dbNames.length > 2) { - ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "Only one dot can be in the name: " + db); - return; - } - - if (Config.isCloudMode()) { - try { - dbName = ((CloudEnv) Env.getCurrentEnv()).analyzeCloudCluster(dbName, ctx); - } catch (DdlException e) { - ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage()); - return; - } - } - - // check catalog and db exists - if (catalogName != null) { - CatalogIf catalogIf = ctx.getEnv().getCatalogMgr().getCatalog(catalogName); - if (catalogIf == null) { - ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "No match catalog in doris: " + db); - return; - } - if (catalogIf.getDbNullable(dbName) == null) { - ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "No match database in doris: " + db); - return; - } - } - try { - if (catalogName != null) { - ctx.getEnv().changeCatalog(ctx, catalogName); - } - Env.getCurrentEnv().changeDb(ctx, dbName); - } catch (DdlException e) { - ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage()); + Optional> res = ConnectContextUtil.initCatalogAndDb(ctx, db); + if (res.isPresent()) { + ctx.getState().setError(res.get().first, res.get().second); return; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/FeNameFormatTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/FeNameFormatTest.java index e4a3b9c9f3c449..ea0b936e422b9e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/FeNameFormatTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/FeNameFormatTest.java @@ -17,10 +17,12 @@ package org.apache.doris.common; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.VariableMgr; import com.google.common.collect.Lists; import org.apache.ivy.util.StringUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -219,6 +221,150 @@ void testUserName() { test(FeNameFormat::checkUserName, alwaysValid, alwaysInvalid, unicodeValid); } + @Test + void testDbName() { + boolean defaultUnicode = VariableMgr.getDefaultSessionVariable().enableUnicodeNameSupport; + boolean defaultNestedNamespace = GlobalVariable.enableNestedNamespace; + List enableUnicode = Lists.newArrayList(false, true); + List enableNestedNamespace = Lists.newArrayList(false, true); + + // Names that are always valid regardless of nested namespace setting + List alwaysValid = Arrays.asList( + "abc123", // ASCII letters + numbers + "A-1_b", // with allowed symbols (-_) + "Z", // single ASCII letter + "a1b2c3", // alphanumeric + "x_y-z", // underscore and hyphen + "test", // letters only + "a-b-c", // multiple hyphens + "a_b", // underscore + "a-1", // hyphen + number + "B2" // uppercase + number + ); + + // Names that are always invalid regardless of settings + List alwaysInvalid = Arrays.asList( + "1abc", // starts with number + "@test", // contains invalid symbol @ + "", // empty string + "a b", // contains space + "abc!", // contains invalid symbol ! + "a\nb", // contains newline + "abc$", // contains invalid symbol $ + "-abc", // starts with hyphen + "_abc", // starts with underscore + "a*b", // contains asterisk + "a#b" // contains hash symbol + ); + + // Names with dots - only valid when nested namespace is enabled + List dotNames = Arrays.asList( + "db1.db2", // database name with dot in middle + "db1.db2.db3", // multiple dots in middle + "a.b.c.d", // multiple segments with dots + "test.prod", // simple dot notation + "system.user.profile" // nested database name + ); + + // Names with dots that are always invalid (start/end with dot, consecutive dots) + List invalidDotNames = Arrays.asList( + ".abc", // starts with dot + "abc.", // ends with dot + ".abc.def", // starts with dot + "abc.def.", // ends with dot + "a..b", // consecutive dots + "a.b.", // ends with dot after valid segment + ".a.b" // starts with dot before valid segment + ); + + // Unicode names that are always valid + List unicodeValid = Lists.newArrayList( + "éclair", // French letters + "über", // German umlaut + "北京", // Chinese characters + "東京123", // Japanese + numbers + "München", // German umlaut + "Beyoncé", // French accent + "αβγ", // Greek letters + "русский", // Cyrillic letters + "øre", // Nordic letter + "ção", // Portuguese letter + "naïve", // French diacritic + "Ḥello", // special diacritic + "ẞig" // German sharp S + ); + + // Unicode names with dots - only valid when both unicode and nested namespace are enabled + List unicodeDotNames = Lists.newArrayList( + "北京.東京", // Chinese and Japanese with dot + "café.système", // French words with dot + "über.München", // German words with dot + "αβγ.русский" // Greek and Cyrillic with dot + ); + + try { + for (Boolean unicode : enableUnicode) { + for (Boolean nestedNamespace : enableNestedNamespace) { + VariableMgr.getDefaultSessionVariable().setEnableUnicodeNameSupport(unicode); + GlobalVariable.enableNestedNamespace = nestedNamespace; + + // Test always valid names + for (String s : alwaysValid) { + ExceptionChecker.expectThrowsNoException(() -> FeNameFormat.checkDbName(s)); + } + + // Test always invalid names + for (String s : alwaysInvalid) { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "name should be invalid: " + s + + " (unicode=" + unicode + ", nested=" + nestedNamespace + ")"); + } + + // Test names with invalid dot patterns (always invalid) + for (String s : invalidDotNames) { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "name should be invalid: " + s + + " (unicode=" + unicode + ", nested=" + nestedNamespace + ")"); + } + + // Test names with dots (valid only when nested namespace is enabled) + for (String s : dotNames) { + if (nestedNamespace) { + ExceptionChecker.expectThrowsNoException(() -> FeNameFormat.checkDbName(s)); + } else { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "name should be invalid when nested namespace is disabled: " + s); + } + } + + // Test unicode names + for (String s : unicodeValid) { + if (unicode) { + ExceptionChecker.expectThrowsNoException(() -> FeNameFormat.checkDbName(s)); + } else { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "unicode name should be invalid when unicode issh bi disabled: " + s); + } + } + + // Test unicode names with dots (valid only when both unicode and nested namespace are enabled) + for (String s : unicodeDotNames) { + if (unicode && nestedNamespace) { + ExceptionChecker.expectThrowsNoException(() -> FeNameFormat.checkDbName(s)); + } else { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "unicode dot name should be invalid: " + s + + " (unicode=" + unicode + ", nested=" + nestedNamespace + ")"); + } + } + } + } + } finally { + VariableMgr.getDefaultSessionVariable().setEnableUnicodeNameSupport(defaultUnicode); + GlobalVariable.enableNestedNamespace = defaultNestedNamespace; + } + } + @Test void testCommonName() { List alwaysValid = Arrays.asList( @@ -245,7 +391,7 @@ void testCommonName() { "_abc", // starts with underscore StringUtils.repeat("a", 65), // exceeds length limit (64) "a*b", // contains asterisk - "a.b", // contains dot (if not allowed) + "a.b", // contains dot (not allowed) "a#b" // contains hash symbol ); List unicodeValid = Lists.newArrayList( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java new file mode 100644 index 00000000000000..3ecdb9ce437086 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.iceberg.catalog.Namespace; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +public class IcebergMetadataOpTest { + + @Test + public void testGetNamespaces() { + Namespace ns = IcebergMetadataOps.getNamespace(Optional.empty(), "db1"); + Assert.assertEquals(1, ns.length()); + + ns = IcebergMetadataOps.getNamespace(Optional.empty(), "db1.db2.db3"); + Assert.assertEquals(3, ns.length()); + + ns = IcebergMetadataOps.getNamespace(Optional.empty(), "db1..db2"); + Assert.assertEquals(2, ns.length()); + + ns = IcebergMetadataOps.getNamespace(Optional.of("p1"), "db1"); + Assert.assertEquals(2, ns.length()); + + ns = IcebergMetadataOps.getNamespace(Optional.of("p1"), ""); + Assert.assertEquals(1, ns.length()); + + ns = IcebergMetadataOps.getNamespace(Optional.empty(), ""); + Assert.assertEquals(0, ns.length()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java index d1199df2ae647a..226f6dd65511b5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java @@ -131,14 +131,14 @@ public void testOAuth2ValidationErrors() { IcebergRestProperties restProps2 = new IcebergRestProperties(props2); Assertions.assertThrows(IllegalArgumentException.class, restProps2::initNormalizeAndCheckProps); - // Test: credential flow without server URI + // Test: credential flow without server URI is ok Map props3 = new HashMap<>(); props3.put("iceberg.rest.uri", "http://localhost:8080"); props3.put("iceberg.rest.security.type", "oauth2"); props3.put("iceberg.rest.oauth2.credential", "client_credentials"); IcebergRestProperties restProps3 = new IcebergRestProperties(props3); - Assertions.assertThrows(IllegalArgumentException.class, restProps3::initNormalizeAndCheckProps); + Assertions.assertDoesNotThrow(restProps3::initNormalizeAndCheckProps); // Test: scope with token (should fail) Map props4 = new HashMap<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java index 0142b285dd775a..18a7117e920a3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java @@ -314,15 +314,6 @@ public void testNegotiateSendFail() throws Exception { Assert.assertFalse(MysqlProto.negotiate(context)); } - @Test - public void testNegotiateInvalidPasswd() throws Exception { - mockChannel("user", true); - mockPassword(false); - mockAccess(); - ConnectContext context = new ConnectContext(streamConnection); - Assert.assertTrue(MysqlProto.negotiate(context)); - } - @Test public void testNegotiateNoUser() throws Exception { mockChannel("", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index 573479a4ec5960..81d41c70bdf447 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -18,6 +18,9 @@ package org.apache.doris.qe; import org.apache.doris.catalog.Env; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.Pair; import org.apache.doris.mysql.MysqlCapability; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.privilege.Auth; @@ -102,7 +105,7 @@ public void testNormal() { // Thread info Assert.assertNotNull(ctx.toThreadInfo(false)); - List row = ctx.toThreadInfo(false).toRow(101, 1000, Optional.empty()); + List row = ctx.toThreadInfo(false).toRow(101, 1000, Optional.of("+08:00")); Assert.assertEquals(14, row.size()); Assert.assertEquals("Yes", row.get(0)); Assert.assertEquals("101", row.get(1)); @@ -297,4 +300,178 @@ public void testResetQueryId() { Assert.assertEquals(queryId2, context.queryId); Assert.assertEquals(queryId, context.lastQueryId); } + + @Test + public void testInitCatalogAndDbSinglePart() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeDb(ctx, "testDb"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, "testDb"); + Assert.assertFalse(result.isPresent()); + } + + @Test + public void testInitCatalogAndDbTwoParts() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeCatalog(ctx, "catalog1"); + minTimes = 0; + env.changeDb(ctx, "testDb"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, "catalog1.testDb"); + Assert.assertFalse(result.isPresent()); + } + + @Test + public void testInitCatalogAndDbMultiplePartsWithNestedNamespaceEnabled() throws Exception { + // Temporarily set the field value + boolean originalValue = GlobalVariable.enableNestedNamespace; + GlobalVariable.enableNestedNamespace = true; + + try { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeCatalog(ctx, "catalog1"); + minTimes = 0; + env.changeDb(ctx, "ns1.ns2.testDb"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, + "catalog1.ns1.ns2.testDb"); + Assert.assertFalse(result.isPresent()); + } finally { + GlobalVariable.enableNestedNamespace = originalValue; + } + } + + @Test + public void testInitCatalogAndDbMultiplePartsWithNestedNamespaceDisabled() throws Exception { + // Ensure GlobalVariable.enableNestedNamespace is false (default) + boolean originalValue = GlobalVariable.enableNestedNamespace; + GlobalVariable.enableNestedNamespace = false; + + try { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, + "catalog1.ns1.ns2.testDb"); + Assert.assertTrue(result.isPresent()); + Assert.assertEquals(ErrorCode.ERR_BAD_DB_ERROR, result.get().first); + Assert.assertTrue(result.get().second.contains("Only one dot can be in the name")); + } finally { + GlobalVariable.enableNestedNamespace = originalValue; + } + } + + @Test + public void testInitCatalogAndDbWithFourPartsNestedNamespaceEnabled() throws Exception { + // Temporarily set GlobalVariable.enableNestedNamespace to be true + boolean originalValue = GlobalVariable.enableNestedNamespace; + GlobalVariable.enableNestedNamespace = true; + + try { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeCatalog(ctx, "catalog1"); + minTimes = 0; + env.changeDb(ctx, "ns1.ns2.ns3.testDb"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, + "catalog1.ns1.ns2.ns3.testDb"); + Assert.assertFalse(result.isPresent()); + } finally { + GlobalVariable.enableNestedNamespace = originalValue; + } + } + + @Test + public void testInitCatalogAndDbWithChangeCatalogException() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeCatalog(ctx, "invalidCatalog"); + result = new DdlException("Catalog not found"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, "invalidCatalog.testDb"); + Assert.assertTrue(result.isPresent()); + Assert.assertTrue(result.get().second.contains("Catalog not found")); + } + + @Test + public void testInitCatalogAndDbWithChangeDbException() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeDb(ctx, "invalidDb"); + result = new DdlException("Database not found"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, "invalidDb"); + Assert.assertTrue(result.isPresent()); + Assert.assertTrue(result.get().second.contains("Database not found")); + } + + @Test + public void testInitCatalogAndDbEmptyString() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeDb(ctx, ""); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, ""); + Assert.assertFalse(result.isPresent()); + } + + @Test + public void testInitCatalogAndDbNullString() { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + // This should cause a NullPointerException when calling split on null + try { + ConnectContextUtil.initCatalogAndDb(ctx, null); + Assert.fail("Expected NullPointerException"); + } catch (NullPointerException e) { + // Expected behavior + } + } } diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out b/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out new file mode 100644 index 00000000000000..4432b08854ca72 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out @@ -0,0 +1,128 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql01 -- + +-- !sql02 -- + +-- !sql03 -- +ns1 + +-- !sql04 -- +ns1.ns2 + +-- !sql05 -- +ns1.ns2.ns3 + +-- !sql06 -- +101 + +-- !sql07 -- +102 + +-- !sql08 -- +103 + +-- !sql09 -- +101 + +-- !sql10 -- +102 + +-- !sql11 -- +101 +102 +103 + +-- !sql12 -- +nested_tbl1 + +-- !sql13 -- +nested_tbl2 + +-- !sql14 -- +nested_tbl3 + +-- !sql15 -- + +-- !sql16 -- +ns1.ns2 + +-- !sql17 -- +ns1.ns2 + +-- !sql18 -- + +-- !sql19 -- + +-- !sql20 -- + +-- !sql21 -- +ns1.ns2 + +-- !sql22 -- +ns1.ns2.ns3 + +-- !sql23 -- + +-- !sql24 -- + +-- !sql25 -- +ns1.ns2 + +-- !sql26 -- +ns1.ns2.ns3 + +-- !sql261 -- +104 + +-- !sql27 -- +ns1.ns2 + +-- !sql28 -- + +-- !sql29 -- +104 + +-- !sql30 -- + +-- !sql31 -- +105 + +-- !sql32 -- + +-- !sql33 -- +nsa + +-- !sql34 -- + +-- !sql35 -- +106 + +-- !sql1001 -- +idb1 + +-- !sql1002 -- +idb1.idb2 + +-- !sql1003 -- +idb1.idb2.idb3 + +-- !sql101 -- +201 + +-- !sql103 -- +202 + +-- !sql104 -- +201 +202 +203 + +-- !sql2001 -- +idb1 + +-- !sql2002 -- +idb1.idb2 + +-- !sql2003 -- +idb1.idb2.idb3 + diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy new file mode 100644 index 00000000000000..f7eb1f5e8b6048 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy @@ -0,0 +1,298 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("iceberg_and_internal_nested_namespace", "p0,external,doris,external_docker,external_docker_doris") { + + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalog_name = "iceberg_nested_namespace" + + sql """drop catalog if exists ${catalog_name}""" + // 1. + // iceberg.rest.nested-namespace-enabled = false + // set global enable_nested_namespace = false + sql """set global enable_nested_namespace=false""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1", + "iceberg.rest.nested-namespace-enabled" = "false" + );""" + + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + logger.info("switched to catalog " + catalog_name) + + // there is already a nested namespace "nested.db1", but can only see "nested" + sql """show tables from `nested`;""" + test { + sql """show tables from `nested.db1`;""" + exception """Unknown database 'nested.db1'""" + } + test { + sql """drop database `nested.db1`;""" + exception """Can't drop database 'nested.db1'; database doesn't exist""" + } + test { + sql """select * from `nested.db1`.tbl1;""" + exception """Database [nested.db1] does not exist""" + } + // can not create nested ns + test { + sql """create database `ns1.ns2`""" + exception """Incorrect database name 'ns1.ns2'""" + } + + // 2. + // iceberg.rest.nested-namespace-enabled = true + // set global enable_nested_namespace = false + sql """set global enable_nested_namespace = false""" + sql """alter catalog ${catalog_name} set properties("iceberg.rest.nested-namespace-enabled" = "true");""" + sql """switch ${catalog_name}""" + // can see the nested ns, with back quote + sql """show tables from `nested`;""" + sql """show tables from `nested.db1`;""" + test { + sql """show tables from nested.db1""" + exception """Unknown catalog 'nested'""" + } + // for "use" stmt, back quote is not necessary + // sql """use ${catalog_name}.nested.db1""" + // sql """use ${catalog_name}.`nested.db1`""" + // can not create nested ns + test { + sql """create database `ns1.ns2`""" + exception """Incorrect database name 'ns1.ns2'""" + } + + // 3. + // iceberg.rest.nested-namespace-enabled = true + // set global enable_nested_namespace = true + sql """set global enable_nested_namespace = true""" + sql """alter catalog ${catalog_name} set properties("iceberg.rest.nested-namespace-enabled" = "true");""" + sql """switch ${catalog_name}""" + // can see the nested ns, with back quote + sql """show tables from `nested`;""" + sql """show tables from `nested.db1`;""" + test { + sql """show tables from nested.db1""" + exception """Unknown catalog 'nested'""" + } + // for "use" stmt, back quote is not necessary + sql """use ${catalog_name}.nested;""" + sql """use ${catalog_name}.`nested.db1`""" + // drop and create nested db1 + sql """drop database if exists `ns1.ns2.ns3` force""" + sql """drop database if exists `ns1.ns2` force""" + sql """drop database if exists `ns1` force""" + qt_sql01 """show databases like 'ns1.ns2.ns3'""" // empty + sql """refresh catalog ${catalog_name}""" + qt_sql02 """show databases like 'ns1.ns2.ns3'""" // empty + + sql """create database `ns1.ns2.ns3`""" + // will see 3 ns, flat + qt_sql03 """show databases like 'ns1'""" // 1 + qt_sql04 """show databases like 'ns1.ns2'""" // 1 + qt_sql05 """show databases like 'ns1.ns2.ns3'""" // 1 + // can create database in each ns + sql """create table ns1.nested_tbl1 (k1 int)"""; + sql """insert into ns1.nested_tbl1 values(101)""" + qt_sql06 """select * from ns1.nested_tbl1""" + + sql """create table `ns1.ns2`.nested_tbl2 (k1 int)"""; + sql """insert into `ns1.ns2`.nested_tbl2 values(102)""" + qt_sql07 """select * from `ns1.ns2`.nested_tbl2""" + + sql """use ${catalog_name}.`ns1.ns2.ns3`""" + sql """create table nested_tbl3 (k1 int)"""; + sql """insert into nested_tbl3 values(103)""" + qt_sql08 """select * from nested_tbl3""" + + // test select column in diff qualified names + qt_sql09 """select ${catalog_name}.ns1.nested_tbl1.k1 from ${catalog_name}.ns1.nested_tbl1""" + qt_sql10 """select ${catalog_name}.`ns1.ns2`.nested_tbl2.k1 from ${catalog_name}.`ns1.ns2`.nested_tbl2""" + sql """use ${catalog_name}.`ns1.ns2`""" + order_qt_sql11 """select ${catalog_name}.ns1.nested_tbl1.k1 from ${catalog_name}.ns1.nested_tbl1 + union all + select k1 from nested_tbl2 + union all + select `ns1.ns2.ns3`.nested_tbl3.k1 from `ns1.ns2.ns3`.nested_tbl3; + """ + // test table exist in each ns + qt_sql12 """show tables from ns1"""; + qt_sql13 """show tables from `ns1.ns2`"""; + qt_sql14 """show tables from `ns1.ns2.ns3`"""; + test { + sql """drop database ns1""" + exception """Namespace ns1 is not empty. 1 tables exist""" + } + test { + sql """drop database `ns1.ns2`""" + exception """Namespace ns1.ns2 is not empty. 1 tables exist""" + } + test { + sql """drop database ${catalog_name}.`ns1.ns2.ns3`""" + exception """Namespace ns1.ns2.ns3 is not empty. 1 tables exist""" + } + // test refresh database and table + sql """refresh database ${catalog_name}.`ns1.ns2`""" + sql """refresh database `ns1.ns2`""" + sql """refresh table ${catalog_name}.`ns1.ns2`.nested_tbl2""" + sql """refresh table `ns1.ns2`.nested_tbl2""" + test { + sql """refresh table `ns1.ns2`.nested_tbl2xxx""" + exception """Table nested_tbl2xxx does not exist in db ns1.ns2""" + } + // drop ns1.ns2 first, we can still see it after refresh, because ns1.ns2.ns3 still exists + sql """drop database `ns1.ns2` force""" + qt_sql15 """show databases like "ns1.ns2"""" // empty + sql """refresh catalog ${catalog_name}""" + qt_sql16 """show databases like "ns1.ns2"""" // 1 + // then we drop ns1.ns2.ns3, after refresh, ns1.ns2 also disappear + sql """drop database `ns1.ns2.ns3` force""" + qt_sql17 """show databases like "ns1.ns2"""" // 1 + qt_sql18 """show databases like "ns1.ns2.ns3"""" // empty + sql """refresh catalog ${catalog_name}""" + qt_sql19 """show databases like "ns1.ns2"""" // empty + qt_sql20 """show databases like "ns1.ns2.ns3"""" // empty + + // recreate ns1.ns2.ns3 + sql """create database `ns1.ns2.ns3`;""" + qt_sql21 """show databases like "ns1.ns2"""" // 1 + qt_sql22 """show databases like "ns1.ns2.ns3"""" // 1 + // drop ns1.ns2.ns3, and ns1.ns2 will disappear too + sql """drop database `ns1.ns2.ns3`""" + sql """refresh catalog ${catalog_name}""" + qt_sql23 """show databases like "ns1.ns2"""" // empty + qt_sql24 """show databases like "ns1.ns2.ns3"""" // empty + + // recreate ns1.ns2.ns3, and create table in ns1.ns2 + sql """create database `ns1.ns2.ns3`;""" + qt_sql25 """show databases like "ns1.ns2"""" // 1 + qt_sql26 """show databases like "ns1.ns2.ns3"""" // 1 + sql """create table `ns1.ns2`.test_table2(k1 int);""" + sql """insert into `ns1.ns2`.test_table2 values(104)""" + qt_sql261 """select * from `ns1.ns2`.test_table2""" + // drop ns1.ns2.ns3, ns1.ns2 will still exist + sql """drop database `ns1.ns2.ns3`""" + sql """refresh catalog ${catalog_name}""" + qt_sql27 """show databases like "ns1.ns2"""" // 1 + qt_sql28 """show databases like "ns1.ns2.ns3"""" // empty + qt_sql29 """select * from `ns1.ns2`.test_table2""" + // drop `ns1.ns2`.test_table2, and then ns1.ns2 will disappeal + sql """drop table `ns1.ns2`.test_table2""" + sql """refresh catalog ${catalog_name}""" + qt_sql30 """show databases like "ns1.ns2"""" // empty + + // test dropping and creating table in nested ns spark created + sql """drop table if exists `nested.db1`.spark_table""" + sql """create table `nested.db1`.spark_table (k1 int)""" + sql """insert into `nested.db1`.spark_table values(105)""" + qt_sql31 """select * from `nested.db1`.spark_table""" + + + // 4. + // iceberg.rest.nested-namespace-enabled = false + // set global enable_nested_namespace = true + sql """set global enable_nested_namespace = true""" + sql """alter catalog ${catalog_name} set properties("iceberg.rest.nested-namespace-enabled" = "false");""" + sql """switch ${catalog_name}""" + // can not see the nested ns + qt_sql32 """show databases like "nested.db1";""" // empty + test { + sql """use ${catalog_name}.`nested.db1`""" + exception """Unknown database 'nested.db1'""" + } + + // can create nested ns, but can not drop because nested ns can not be seen + test { + sql """drop database `nested.db1`""" + exception """Can't drop database 'nested.db1'; database doesn't exist""" + } + sql """create database if not exists `nsa.nsb`""" + sql """create database if not exists `nsa.nsb.nsc`""" + // can only see nsa + qt_sql33 """show databases like "nsa"""" // 1 + qt_sql34 """show databases like "nsa.nsb"""" // empty + // can create and drop table in nsa + sql """drop table if exists nsa.nsa_tbl1""" + sql """create table nsa.nsa_tbl1 (k1 int);""" + sql """insert into nsa.nsa_tbl1 values(106)""" + qt_sql35 """select * from nsa.nsa_tbl1""" + sql """drop table nsa.nsa_tbl1""" + test { + sql """select * from nsa.nsa_tbl1""" + exception """Table [nsa_tbl1] does not exist in database [nsa]""" + } + + // 5. test internal + sql """switch internal""" + sql """set global enable_nested_namespace = true""" + // create nested ns + sql """drop database if exists `idb1.idb2.idb3`""" + sql """drop database if exists `idb1.idb2`""" + sql """drop database if exists `idb1`""" + sql """create database `idb1`""" + sql """create database `idb1.idb2`""" + sql """create database `idb1.idb2.idb3`""" + qt_sql1001 """show databases like "idb1"""" + qt_sql1002 """show databases like "idb1.idb2"""" + qt_sql1003 """show databases like "idb1.idb2.idb3"""" + + // create table + sql """create table idb1.itbl1 (k1 int) properties("replication_num" = "1")""" + sql """create table `idb1.idb2`.itbl2 (k1 int) properties("replication_num" = "1")""" + sql """use internal.`idb1.idb2.idb3`""" + sql """create table itbl3 (k1 int) properties("replication_num" = "1")""" + + // insert + sql """insert into idb1.itbl1 values(201)""" + sql """insert into `idb1.idb2`.itbl2 values(202)""" + sql """use internal.`idb1.idb2.idb3`""" + sql """insert into itbl3 values(203)""" + + // query + qt_sql101 """select * from idb1.itbl1""" + qt_sql103 """select `idb1.idb2`.itbl2.k1 from `idb1.idb2`.itbl2""" + sql """use internal.`idb1.idb2.idb3`""" + order_qt_sql104 """select `idb1.idb2`.itbl2.k1 from `idb1.idb2`.itbl2 + union all + select idb1.itbl1.k1 from idb1.itbl1 + union all + select itbl3.k1 from itbl3 + """ + // disable + sql """set global enable_nested_namespace = false""" + // still can see nested ns + qt_sql2001 """show databases like "idb1"""" + qt_sql2002 """show databases like "idb1.idb2"""" + qt_sql2003 """show databases like "idb1.idb2.idb3"""" + + sql """ unset global variable enable_nested_namespace;""" +}