diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run20.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run20.sql new file mode 100644 index 00000000000000..dba9680f456186 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run20.sql @@ -0,0 +1,2 @@ +create database if not exists nested.db1; + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index f4e2a8309cf919..48ab0550501c8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3375,7 +3375,7 @@ public void createDb(CreateDatabaseCommand command) throws DdlException { if (StringUtils.isEmpty(command.getCtlName())) { catalogIf = getCurrentCatalog(); } else { - catalogIf = catalogMgr.getCatalog(command.getCtlName()); + catalogIf = catalogMgr.getCatalogOrDdlException(command.getCtlName()); } catalogIf.createDb(command.getDbName(), command.isIfNotExists(), command.getProperties()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index 82db6ac659b287..9917f7c7030279 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -24,6 +24,7 @@ import org.apache.doris.mysql.privilege.Role; import org.apache.doris.mysql.privilege.RoleManager; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.VariableMgr; import com.google.common.base.Strings; @@ -53,6 +54,10 @@ public class FeNameFormat { public static final String TEMPORARY_TABLE_SIGN = "_#TEMP#_"; + private static final String NESTED_DB_NAME_REGEX = "^[a-zA-Z][a-zA-Z0-9\\-_]*(\\.([a-zA-Z0-9\\-_]+))*$"; + private static final String NESTED_UNICODE_DB_NAME_REGEX + = "^[a-zA-Z\\p{L}][a-zA-Z0-9\\-_\\p{L}]*(\\.([a-zA-Z0-9\\-_\\p{L}]+))*$"; + public static void checkCatalogName(String catalogName) throws AnalysisException { if (!InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalogName) && (Strings.isNullOrEmpty(catalogName) || !catalogName.matches(getCommonNameRegex()))) { @@ -61,7 +66,10 @@ public static void checkCatalogName(String catalogName) throws AnalysisException } public static void checkDbName(String dbName) throws AnalysisException { - if (Strings.isNullOrEmpty(dbName) || !dbName.matches(getCommonNameRegex())) { + if (Strings.isNullOrEmpty(dbName)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName); + } + if (!dbName.matches(getDbNameRegex())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName); } } @@ -257,6 +265,22 @@ public static String getCommonNameRegex() { } } + public static String getDbNameRegex() { + if (GlobalVariable.enableNestedNamespace) { + if (FeNameFormat.isEnableUnicodeNameSupport()) { + return NESTED_UNICODE_DB_NAME_REGEX; + } else { + return NESTED_DB_NAME_REGEX; + } + } else { + if (FeNameFormat.isEnableUnicodeNameSupport()) { + return UNICODE_COMMON_NAME_REGEX; + } else { + return COMMON_NAME_REGEX; + } + } + } + public static String getOutfileSuccessFileNameRegex() { if (FeNameFormat.isEnableUnicodeNameSupport()) { return UNICODE_UNDERSCORE_COMMON_NAME_REGEX; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index f57392a4f4cf0a..d722183caf9100 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -33,6 +33,8 @@ import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.operations.ExternalMetadataOps; +import org.apache.doris.datasource.property.metastore.IcebergRestProperties; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions; import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; @@ -54,12 +56,14 @@ import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.view.View; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.Collections; @@ -68,6 +72,7 @@ import java.util.Optional; import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; +import java.util.stream.Stream; public class IcebergMetadataOps implements ExternalMetadataOps { @@ -129,15 +134,36 @@ public boolean databaseExist(String dbName) { public List listDatabaseNames() { try { - return executionAuthenticator.execute(() -> nsCatalog.listNamespaces(getNamespace()) - .stream() - .map(n -> n.level(n.length() - 1)) - .collect(Collectors.toList())); + return executionAuthenticator.execute(() -> listNestedNamespaces(getNamespace())); } catch (Exception e) { throw new RuntimeException("Failed to list database names, error message is:" + e.getMessage(), e); } } + @NotNull + private List listNestedNamespaces(Namespace parentNs) { + // Handle nested namespaces for Iceberg REST catalog, + // only if "iceberg.rest.nested-namespace-enabled" is true. + if (dorisCatalog instanceof IcebergRestExternalCatalog) { + IcebergRestExternalCatalog restCatalog = (IcebergRestExternalCatalog) dorisCatalog; + MetastoreProperties metaProps = restCatalog.getCatalogProperty().getMetastoreProperties(); + if (metaProps instanceof IcebergRestProperties + && ((IcebergRestProperties) metaProps).isIcebergRestNestedNamespaceEnabled()) { + return nsCatalog.listNamespaces(parentNs) + .stream() + .flatMap(childNs -> Stream.concat( + Stream.of(childNs.toString()), + listNestedNamespaces(childNs).stream() + )).collect(Collectors.toList()); + } + } + + return nsCatalog.listNamespaces(parentNs) + .stream() + .map(n -> n.level(n.length() - 1)) + .collect(Collectors.toList()); + } + @Override public List listTableNames(String dbName) { try { @@ -209,7 +235,7 @@ private boolean performCreateDb(String dbName, boolean ifNotExists, Map { - preformDropDb(dbName, ifExists, force); + performDropDb(dbName, ifExists, force); return null; }); } catch (Exception e) { @@ -218,7 +244,7 @@ public void dropDbImpl(String dbName, boolean ifExists, boolean force) throws Dd } } - private void preformDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { + private void performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { ExternalDatabase dorisDb = dorisCatalog.getDbNullable(dbName); if (dorisDb == null) { if (ifExists) { @@ -229,21 +255,27 @@ private void preformDropDb(String dbName, boolean ifExists, boolean force) throw } } if (force) { - // try to drop all tables in the database - List remoteTableNames = listTableNames(dorisDb.getRemoteName()); - for (String remoteTableName : remoteTableNames) { - performDropTable(dorisDb.getRemoteName(), remoteTableName, true); - } - if (!remoteTableNames.isEmpty()) { - LOG.info("drop database[{}] with force, drop all tables, num: {}", dbName, remoteTableNames.size()); - } - // try to drop all views in the database - List remoteViewNames = listViewNames(dorisDb.getRemoteName()); - for (String remoteViewName : remoteViewNames) { - performDropView(dorisDb.getRemoteName(), remoteViewName); - } - if (!remoteViewNames.isEmpty()) { - LOG.info("drop database[{}] with force, drop all views, num: {}", dbName, remoteViewNames.size()); + try { + // try to drop all tables in the database + List remoteTableNames = listTableNames(dorisDb.getRemoteName()); + for (String remoteTableName : remoteTableNames) { + performDropTable(dorisDb.getRemoteName(), remoteTableName, true); + } + if (!remoteTableNames.isEmpty()) { + LOG.info("drop database[{}] with force, drop all tables, num: {}", dbName, remoteTableNames.size()); + } + // try to drop all views in the database + List remoteViewNames = listViewNames(dorisDb.getRemoteName()); + for (String remoteViewName : remoteViewNames) { + performDropView(dorisDb.getRemoteName(), remoteViewName); + } + if (!remoteViewNames.isEmpty()) { + LOG.info("drop database[{}] with force, drop all views, num: {}", dbName, remoteViewNames.size()); + } + } catch (NoSuchNamespaceException e) { + // just ignore + LOG.info("drop database[{}] force which does not exist", dbName); + return; } } nsCatalog.dropNamespace(getNamespace(dorisDb.getRemoteName())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java index ed9abc84e9fe9d..faf5d736af1523 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java @@ -110,9 +110,8 @@ public class IcebergRestProperties extends AbstractIcebergProperties { @ConnectorProperty(names = {"iceberg.rest.nested-namespace-enabled"}, required = false, - supported = false, description = "Enable nested namespace for the iceberg rest catalog service.") - private String icebergRestNestedNamespaceEnabled = "true"; + private String icebergRestNestedNamespaceEnabled = "false"; @ConnectorProperty(names = {"iceberg.rest.case-insensitive-name-matching"}, required = false, @@ -306,6 +305,10 @@ public boolean isIcebergRestVendedCredentialsEnabled() { return Boolean.parseBoolean(icebergRestVendedCredentialsEnabled); } + public boolean isIcebergRestNestedNamespaceEnabled() { + return Boolean.parseBoolean(icebergRestNestedNamespaceEnabled); + } + /** * Unified method to configure FileIO properties for Iceberg catalog. * This method handles all storage types (HDFS, S3, MinIO, etc.) and populates diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java index 3b0a5bd8c442df..8a570e6bc9947e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java @@ -18,14 +18,15 @@ package org.apache.doris.mysql; import org.apache.doris.catalog.Env; -import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.Pair; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ConnectContextUtil; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -33,6 +34,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Optional; // MySQL protocol util public class MysqlProto { @@ -224,42 +226,9 @@ public static boolean negotiate(ConnectContext context) throws IOException { // set database String db = authPacket.getDb(); if (!Strings.isNullOrEmpty(db)) { - String catalogName = null; - String dbName = null; - String[] dbNames = db.split("\\."); - if (dbNames.length == 1) { - dbName = db; - } else if (dbNames.length == 2) { - catalogName = dbNames[0]; - dbName = dbNames[1]; - } else if (dbNames.length > 2) { - context.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "Only one dot can be in the name: " + db); - sendResponsePacket(context); - return false; - } - - // mysql -d - if (Config.isCloudMode()) { - try { - dbName = ((CloudEnv) Env.getCurrentEnv()).analyzeCloudCluster(dbName, context); - } catch (DdlException e) { - context.getState().setError(e.getMysqlErrorCode(), e.getMessage()); - sendResponsePacket(context); - return false; - } - - if (dbName == null || dbName.isEmpty()) { - return true; - } - } - - try { - if (catalogName != null) { - context.getEnv().changeCatalog(context, catalogName); - } - Env.getCurrentEnv().changeDb(context, dbName); - } catch (DdlException e) { - context.getState().setError(e.getMysqlErrorCode(), e.getMessage()); + Optional> res = ConnectContextUtil.initCatalogAndDb(context, db); + if (res.isPresent()) { + context.getState().setError(res.get().first, res.get().second); sendResponsePacket(context); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index b2665ebdd42914..828285c66dff3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1043,6 +1043,7 @@ import org.apache.doris.policy.FilterType; import org.apache.doris.policy.PolicyTypeEnum; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.SqlModeHelper; import org.apache.doris.resource.workloadschedpolicy.WorkloadActionMeta; import org.apache.doris.resource.workloadschedpolicy.WorkloadConditionMeta; @@ -5270,15 +5271,21 @@ public RefreshDatabaseCommand visitRefreshDatabase(RefreshDatabaseContext ctx) { if (size == 0) { throw new ParseException("database name can't be empty"); } - String dbName = parts.get(size - 1); // [db]. if (size == 1) { - return new RefreshDatabaseCommand(dbName, properties); + return new RefreshDatabaseCommand(parts.get(0), properties); } else if (parts.size() == 2) { // [ctl,db]. - return new RefreshDatabaseCommand(parts.get(0), dbName, properties); + return new RefreshDatabaseCommand(parts.get(0), parts.get(1), properties); + } else { + if (GlobalVariable.enableNestedNamespace) { + // use the first part as catalog name, the rest part as db name + return new RefreshDatabaseCommand(parts.get(0), + String.join(".", parts.subList(1, size)), properties); + } else { + throw new ParseException("Only one dot can be in the name: " + String.join(".", parts)); + } } - throw new ParseException("Only one dot can be in the name: " + String.join(".", parts)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropCatalogCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropCatalogCommand.java index 034ecb1053a5f6..8a0cb73f75e1b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropCatalogCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropCatalogCommand.java @@ -50,7 +50,7 @@ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { Util.checkCatalogAllRules(catalogName); if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { - throw new AnalysisException("Internal catalog can't be drop."); + throw new AnalysisException("Internal catalog can't be dropped."); } if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv( diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java index b0a705599cab86..48c7744b570909 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java @@ -19,8 +19,17 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.Util; import org.apache.doris.nereids.StatementContext; +import java.util.Optional; +import java.util.stream.Stream; + public class ConnectContextUtil { // Sometimes it's necessary to parse SQL, but not in a user thread where no ConnectContext exists. @@ -39,4 +48,50 @@ public static ConnectContext getDummyCtx(String dbName) { ctx.setThreadLocalInfo(); return ctx; } + + public static Optional> initCatalogAndDb(ConnectContext ctx, String fullDbName) { + String catalogName = null; + String dbName = null; + String[] dbNames = fullDbName.split("\\."); + if (dbNames.length == 1) { + dbName = fullDbName; + } else if (dbNames.length == 2) { + catalogName = dbNames[0]; + dbName = dbNames[1]; + } else if (dbNames.length > 2) { + if (GlobalVariable.enableNestedNamespace) { + // use the first part as catalog name, the rest part as db name + catalogName = dbNames[0]; + dbName = Stream.of(dbNames).skip(1).reduce((a, b) -> a + "." + b).get(); + } else { + return Optional.of( + Pair.of(ErrorCode.ERR_BAD_DB_ERROR, "Only one dot can be in the name: " + fullDbName)); + } + } + + // mysql client + if (Config.isCloudMode()) { + try { + dbName = ((CloudEnv) ctx.getEnv()).analyzeCloudCluster(dbName, ctx); + } catch (DdlException e) { + return Optional.of(Pair.of(e.getMysqlErrorCode(), e.getMessage())); + } + if (dbName == null || dbName.isEmpty()) { + return Optional.empty(); + } + } + + try { + if (catalogName != null) { + ctx.getEnv().changeCatalog(ctx, catalogName); + } + ctx.getEnv().changeDb(ctx, dbName); + } catch (DdlException e) { + return Optional.of(Pair.of(e.getMysqlErrorCode(), e.getMessage())); + } catch (Throwable t) { + return Optional.of(Pair.of(ErrorCode.ERR_INTERNAL_ERROR, Util.getRootCauseMessage(t))); + } + ctx.getState().setOk(); + return Optional.empty(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index f285fd6a811580..1f9e0bb49a0184 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -25,21 +25,18 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; -import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.qe.ComputeGroupException; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.ConnectionException; -import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.SqlUtils; -import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.metric.MetricRepo; @@ -118,45 +115,10 @@ public boolean isHandleQueryInFe() { // change current database of this session. protected void handleInitDb(String fullDbName) { - String catalogName = null; - String dbName = null; - String[] dbNames = fullDbName.split("\\."); - if (dbNames.length == 1) { - dbName = fullDbName; - } else if (dbNames.length == 2) { - catalogName = dbNames[0]; - dbName = dbNames[1]; - } else if (dbNames.length > 2) { - ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "Only one dot can be in the name: " + fullDbName); - return; - } - - // mysql client - if (Config.isCloudMode()) { - try { - dbName = ((CloudEnv) ctx.getEnv()).analyzeCloudCluster(dbName, ctx); - } catch (DdlException e) { - ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage()); - return; - } - if (dbName == null || dbName.isEmpty()) { - return; - } + Optional> res = ConnectContextUtil.initCatalogAndDb(ctx, fullDbName); + if (res.isPresent()) { + ctx.getState().setError(res.get().first, res.get().second); } - - try { - if (catalogName != null) { - ctx.getEnv().changeCatalog(ctx, catalogName); - } - ctx.getEnv().changeDb(ctx, dbName); - } catch (DdlException e) { - ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage()); - return; - } catch (Throwable t) { - ctx.getState().setError(ErrorCode.ERR_INTERNAL_ERROR, Util.getRootCauseMessage(t)); - return; - } - ctx.getState().setOk(); } // set killed flag diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index 31901c76c4a0e2..ff2b423b8a18bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -79,6 +79,8 @@ public final class GlobalVariable { public static final String ENABLE_FETCH_ICEBERG_STATS = "enable_fetch_iceberg_stats"; + public static final String ENABLE_NESTED_NAMESPACE = "enable_nested_namespace"; + public static final String ENABLE_ANSI_QUERY_ORGANIZATION_BEHAVIOR = "enable_ansi_query_organization_behavior"; public static final String ENABLE_NEW_TYPE_COERCION_BEHAVIOR @@ -239,6 +241,16 @@ public final class GlobalVariable { + " a numeric type and precision cannot be determined, the DECIMAL type is preferred."}) public static boolean enableNewTypeCoercionBehavior = true; + @VariableMgr.VarAttr(name = ENABLE_NESTED_NAMESPACE, flag = VariableMgr.GLOBAL, + description = { + "是否允许访问 `ns1.ns2` 这种类型的 database。当前仅适用于 External Catalog 中映射 Database 并访问。" + + "不支持创建。", + "Whether to allow accessing databases of the form `ns1.ns2`. " + + "Currently, this only applies to mapping databases in " + + "External Catalogs and accessing them. " + + "Creation is not supported."}) + public static boolean enableNestedNamespace = false; + // Don't allow creating instance. private GlobalVariable() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 3b309136990f95..50d3bed474a98f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -21,14 +21,11 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MysqlColType; -import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.common.AuthenticationException; -import org.apache.doris.common.Config; import org.apache.doris.common.ConnectionException; -import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.common.Pair; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlCommand; @@ -58,6 +55,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * Process one mysql connection, receive one packet, process, send one packet. @@ -355,47 +353,9 @@ private void handleChangeUser() throws IOException { if (Strings.isNullOrEmpty(db)) { ctx.changeDefaultCatalog(InternalCatalog.INTERNAL_CATALOG_NAME); } else { - String catalogName = null; - String dbName = null; - String[] dbNames = db.split("\\."); - if (dbNames.length == 1) { - dbName = db; - } else if (dbNames.length == 2) { - catalogName = dbNames[0]; - dbName = dbNames[1]; - } else if (dbNames.length > 2) { - ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "Only one dot can be in the name: " + db); - return; - } - - if (Config.isCloudMode()) { - try { - dbName = ((CloudEnv) Env.getCurrentEnv()).analyzeCloudCluster(dbName, ctx); - } catch (DdlException e) { - ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage()); - return; - } - } - - // check catalog and db exists - if (catalogName != null) { - CatalogIf catalogIf = ctx.getEnv().getCatalogMgr().getCatalog(catalogName); - if (catalogIf == null) { - ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "No match catalog in doris: " + db); - return; - } - if (catalogIf.getDbNullable(dbName) == null) { - ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "No match database in doris: " + db); - return; - } - } - try { - if (catalogName != null) { - ctx.getEnv().changeCatalog(ctx, catalogName); - } - Env.getCurrentEnv().changeDb(ctx, dbName); - } catch (DdlException e) { - ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage()); + Optional> res = ConnectContextUtil.initCatalogAndDb(ctx, db); + if (res.isPresent()) { + ctx.getState().setError(res.get().first, res.get().second); return; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/FeNameFormatTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/FeNameFormatTest.java index 124604bff8d5cd..336c693d82f31e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/FeNameFormatTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/FeNameFormatTest.java @@ -17,6 +17,7 @@ package org.apache.doris.common; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.VariableMgr; import com.google.common.collect.Lists; @@ -235,6 +236,150 @@ void testUserName() { test(FeNameFormat::checkUserName, alwaysValid, alwaysInvalid, unicodeValid); } + @Test + void testDbName() { + boolean defaultUnicode = VariableMgr.getDefaultSessionVariable().enableUnicodeNameSupport; + boolean defaultNestedNamespace = GlobalVariable.enableNestedNamespace; + List enableUnicode = Lists.newArrayList(false, true); + List enableNestedNamespace = Lists.newArrayList(false, true); + + // Names that are always valid regardless of nested namespace setting + List alwaysValid = Arrays.asList( + "abc123", // ASCII letters + numbers + "A-1_b", // with allowed symbols (-_) + "Z", // single ASCII letter + "a1b2c3", // alphanumeric + "x_y-z", // underscore and hyphen + "test", // letters only + "a-b-c", // multiple hyphens + "a_b", // underscore + "a-1", // hyphen + number + "B2" // uppercase + number + ); + + // Names that are always invalid regardless of settings + List alwaysInvalid = Arrays.asList( + "1abc", // starts with number + "@test", // contains invalid symbol @ + "", // empty string + "a b", // contains space + "abc!", // contains invalid symbol ! + "a\nb", // contains newline + "abc$", // contains invalid symbol $ + "-abc", // starts with hyphen + "_abc", // starts with underscore + "a*b", // contains asterisk + "a#b" // contains hash symbol + ); + + // Names with dots - only valid when nested namespace is enabled + List dotNames = Arrays.asList( + "db1.db2", // database name with dot in middle + "db1.db2.db3", // multiple dots in middle + "a.b.c.d", // multiple segments with dots + "test.prod", // simple dot notation + "system.user.profile" // nested database name + ); + + // Names with dots that are always invalid (start/end with dot, consecutive dots) + List invalidDotNames = Arrays.asList( + ".abc", // starts with dot + "abc.", // ends with dot + ".abc.def", // starts with dot + "abc.def.", // ends with dot + "a..b", // consecutive dots + "a.b.", // ends with dot after valid segment + ".a.b" // starts with dot before valid segment + ); + + // Unicode names that are always valid + List unicodeValid = Lists.newArrayList( + "éclair", // French letters + "über", // German umlaut + "北京", // Chinese characters + "東京123", // Japanese + numbers + "München", // German umlaut + "Beyoncé", // French accent + "αβγ", // Greek letters + "русский", // Cyrillic letters + "øre", // Nordic letter + "ção", // Portuguese letter + "naïve", // French diacritic + "Ḥello", // special diacritic + "ẞig" // German sharp S + ); + + // Unicode names with dots - only valid when both unicode and nested namespace are enabled + List unicodeDotNames = Lists.newArrayList( + "北京.東京", // Chinese and Japanese with dot + "café.système", // French words with dot + "über.München", // German words with dot + "αβγ.русский" // Greek and Cyrillic with dot + ); + + try { + for (Boolean unicode : enableUnicode) { + for (Boolean nestedNamespace : enableNestedNamespace) { + VariableMgr.getDefaultSessionVariable().setEnableUnicodeNameSupport(unicode); + GlobalVariable.enableNestedNamespace = nestedNamespace; + + // Test always valid names + for (String s : alwaysValid) { + ExceptionChecker.expectThrowsNoException(() -> FeNameFormat.checkDbName(s)); + } + + // Test always invalid names + for (String s : alwaysInvalid) { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "name should be invalid: " + s + + " (unicode=" + unicode + ", nested=" + nestedNamespace + ")"); + } + + // Test names with invalid dot patterns (always invalid) + for (String s : invalidDotNames) { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "name should be invalid: " + s + + " (unicode=" + unicode + ", nested=" + nestedNamespace + ")"); + } + + // Test names with dots (valid only when nested namespace is enabled) + for (String s : dotNames) { + if (nestedNamespace) { + ExceptionChecker.expectThrowsNoException(() -> FeNameFormat.checkDbName(s)); + } else { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "name should be invalid when nested namespace is disabled: " + s); + } + } + + // Test unicode names + for (String s : unicodeValid) { + if (unicode) { + ExceptionChecker.expectThrowsNoException(() -> FeNameFormat.checkDbName(s)); + } else { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "unicode name should be invalid when unicode issh bi disabled: " + s); + } + } + + // Test unicode names with dots (valid only when both unicode and nested namespace are enabled) + for (String s : unicodeDotNames) { + if (unicode && nestedNamespace) { + ExceptionChecker.expectThrowsNoException(() -> FeNameFormat.checkDbName(s)); + } else { + Assertions.assertThrowsExactly(AnalysisException.class, () -> FeNameFormat.checkDbName(s), + "unicode dot name should be invalid: " + s + + " (unicode=" + unicode + ", nested=" + nestedNamespace + ")"); + } + } + } + } + } finally { + VariableMgr.getDefaultSessionVariable().setEnableUnicodeNameSupport(defaultUnicode); + GlobalVariable.enableNestedNamespace = defaultNestedNamespace; + } + } + @Test void testCommonName() { List alwaysValid = Arrays.asList( @@ -261,7 +406,7 @@ void testCommonName() { "_abc", // starts with underscore StringUtils.repeat("a", 65), // exceeds length limit (64) "a*b", // contains asterisk - "a.b", // contains dot (if not allowed) + "a.b", // contains dot (not allowed) "a#b" // contains hash symbol ); List unicodeValid = Lists.newArrayList( diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java index 0142b285dd775a..18a7117e920a3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java @@ -314,15 +314,6 @@ public void testNegotiateSendFail() throws Exception { Assert.assertFalse(MysqlProto.negotiate(context)); } - @Test - public void testNegotiateInvalidPasswd() throws Exception { - mockChannel("user", true); - mockPassword(false); - mockAccess(); - ConnectContext context = new ConnectContext(streamConnection); - Assert.assertTrue(MysqlProto.negotiate(context)); - } - @Test public void testNegotiateNoUser() throws Exception { mockChannel("", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index cc522138ca0ebb..874c30e9cbe082 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -19,6 +19,9 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.Pair; import org.apache.doris.mysql.MysqlCapability; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.privilege.Auth; @@ -103,7 +106,7 @@ public void testNormal() { // Thread info Assert.assertNotNull(ctx.toThreadInfo(false)); - List row = ctx.toThreadInfo(false).toRow(101, 1000, Optional.empty()); + List row = ctx.toThreadInfo(false).toRow(101, 1000, Optional.of("+08:00")); Assert.assertEquals(15, row.size()); Assert.assertEquals("Yes", row.get(0)); Assert.assertEquals("101", row.get(1)); @@ -298,4 +301,178 @@ public void testResetQueryId() { Assert.assertEquals(queryId2, context.queryId); Assert.assertEquals(queryId, context.lastQueryId); } + + @Test + public void testInitCatalogAndDbSinglePart() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeDb(ctx, "testDb"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, "testDb"); + Assert.assertFalse(result.isPresent()); + } + + @Test + public void testInitCatalogAndDbTwoParts() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeCatalog(ctx, "catalog1"); + minTimes = 0; + env.changeDb(ctx, "testDb"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, "catalog1.testDb"); + Assert.assertFalse(result.isPresent()); + } + + @Test + public void testInitCatalogAndDbMultiplePartsWithNestedNamespaceEnabled() throws Exception { + // Temporarily set the field value + boolean originalValue = GlobalVariable.enableNestedNamespace; + GlobalVariable.enableNestedNamespace = true; + + try { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeCatalog(ctx, "catalog1"); + minTimes = 0; + env.changeDb(ctx, "ns1.ns2.testDb"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, + "catalog1.ns1.ns2.testDb"); + Assert.assertFalse(result.isPresent()); + } finally { + GlobalVariable.enableNestedNamespace = originalValue; + } + } + + @Test + public void testInitCatalogAndDbMultiplePartsWithNestedNamespaceDisabled() throws Exception { + // Ensure GlobalVariable.enableNestedNamespace is false (default) + boolean originalValue = GlobalVariable.enableNestedNamespace; + GlobalVariable.enableNestedNamespace = false; + + try { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, + "catalog1.ns1.ns2.testDb"); + Assert.assertTrue(result.isPresent()); + Assert.assertEquals(ErrorCode.ERR_BAD_DB_ERROR, result.get().first); + Assert.assertTrue(result.get().second.contains("Only one dot can be in the name")); + } finally { + GlobalVariable.enableNestedNamespace = originalValue; + } + } + + @Test + public void testInitCatalogAndDbWithFourPartsNestedNamespaceEnabled() throws Exception { + // Temporarily set GlobalVariable.enableNestedNamespace to be true + boolean originalValue = GlobalVariable.enableNestedNamespace; + GlobalVariable.enableNestedNamespace = true; + + try { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeCatalog(ctx, "catalog1"); + minTimes = 0; + env.changeDb(ctx, "ns1.ns2.ns3.testDb"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, + "catalog1.ns1.ns2.ns3.testDb"); + Assert.assertFalse(result.isPresent()); + } finally { + GlobalVariable.enableNestedNamespace = originalValue; + } + } + + @Test + public void testInitCatalogAndDbWithChangeCatalogException() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeCatalog(ctx, "invalidCatalog"); + result = new DdlException("Catalog not found"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, "invalidCatalog.testDb"); + Assert.assertTrue(result.isPresent()); + Assert.assertTrue(result.get().second.contains("Catalog not found")); + } + + @Test + public void testInitCatalogAndDbWithChangeDbException() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeDb(ctx, "invalidDb"); + result = new DdlException("Database not found"); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, "invalidDb"); + Assert.assertTrue(result.isPresent()); + Assert.assertTrue(result.get().second.contains("Database not found")); + } + + @Test + public void testInitCatalogAndDbEmptyString() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + new Expectations() { + { + env.changeDb(ctx, ""); + minTimes = 0; + } + }; + + Optional> result = ConnectContextUtil.initCatalogAndDb(ctx, ""); + Assert.assertFalse(result.isPresent()); + } + + @Test + public void testInitCatalogAndDbNullString() { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(env); + + // This should cause a NullPointerException when calling split on null + try { + ConnectContextUtil.initCatalogAndDb(ctx, null); + Assert.fail("Expected NullPointerException"); + } catch (NullPointerException e) { + // Expected behavior + } + } } diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out b/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out new file mode 100644 index 00000000000000..4432b08854ca72 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out @@ -0,0 +1,128 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql01 -- + +-- !sql02 -- + +-- !sql03 -- +ns1 + +-- !sql04 -- +ns1.ns2 + +-- !sql05 -- +ns1.ns2.ns3 + +-- !sql06 -- +101 + +-- !sql07 -- +102 + +-- !sql08 -- +103 + +-- !sql09 -- +101 + +-- !sql10 -- +102 + +-- !sql11 -- +101 +102 +103 + +-- !sql12 -- +nested_tbl1 + +-- !sql13 -- +nested_tbl2 + +-- !sql14 -- +nested_tbl3 + +-- !sql15 -- + +-- !sql16 -- +ns1.ns2 + +-- !sql17 -- +ns1.ns2 + +-- !sql18 -- + +-- !sql19 -- + +-- !sql20 -- + +-- !sql21 -- +ns1.ns2 + +-- !sql22 -- +ns1.ns2.ns3 + +-- !sql23 -- + +-- !sql24 -- + +-- !sql25 -- +ns1.ns2 + +-- !sql26 -- +ns1.ns2.ns3 + +-- !sql261 -- +104 + +-- !sql27 -- +ns1.ns2 + +-- !sql28 -- + +-- !sql29 -- +104 + +-- !sql30 -- + +-- !sql31 -- +105 + +-- !sql32 -- + +-- !sql33 -- +nsa + +-- !sql34 -- + +-- !sql35 -- +106 + +-- !sql1001 -- +idb1 + +-- !sql1002 -- +idb1.idb2 + +-- !sql1003 -- +idb1.idb2.idb3 + +-- !sql101 -- +201 + +-- !sql103 -- +202 + +-- !sql104 -- +201 +202 +203 + +-- !sql2001 -- +idb1 + +-- !sql2002 -- +idb1.idb2 + +-- !sql2003 -- +idb1.idb2.idb3 + diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy new file mode 100644 index 00000000000000..f7eb1f5e8b6048 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy @@ -0,0 +1,298 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("iceberg_and_internal_nested_namespace", "p0,external,doris,external_docker,external_docker_doris") { + + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalog_name = "iceberg_nested_namespace" + + sql """drop catalog if exists ${catalog_name}""" + // 1. + // iceberg.rest.nested-namespace-enabled = false + // set global enable_nested_namespace = false + sql """set global enable_nested_namespace=false""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1", + "iceberg.rest.nested-namespace-enabled" = "false" + );""" + + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + logger.info("switched to catalog " + catalog_name) + + // there is already a nested namespace "nested.db1", but can only see "nested" + sql """show tables from `nested`;""" + test { + sql """show tables from `nested.db1`;""" + exception """Unknown database 'nested.db1'""" + } + test { + sql """drop database `nested.db1`;""" + exception """Can't drop database 'nested.db1'; database doesn't exist""" + } + test { + sql """select * from `nested.db1`.tbl1;""" + exception """Database [nested.db1] does not exist""" + } + // can not create nested ns + test { + sql """create database `ns1.ns2`""" + exception """Incorrect database name 'ns1.ns2'""" + } + + // 2. + // iceberg.rest.nested-namespace-enabled = true + // set global enable_nested_namespace = false + sql """set global enable_nested_namespace = false""" + sql """alter catalog ${catalog_name} set properties("iceberg.rest.nested-namespace-enabled" = "true");""" + sql """switch ${catalog_name}""" + // can see the nested ns, with back quote + sql """show tables from `nested`;""" + sql """show tables from `nested.db1`;""" + test { + sql """show tables from nested.db1""" + exception """Unknown catalog 'nested'""" + } + // for "use" stmt, back quote is not necessary + // sql """use ${catalog_name}.nested.db1""" + // sql """use ${catalog_name}.`nested.db1`""" + // can not create nested ns + test { + sql """create database `ns1.ns2`""" + exception """Incorrect database name 'ns1.ns2'""" + } + + // 3. + // iceberg.rest.nested-namespace-enabled = true + // set global enable_nested_namespace = true + sql """set global enable_nested_namespace = true""" + sql """alter catalog ${catalog_name} set properties("iceberg.rest.nested-namespace-enabled" = "true");""" + sql """switch ${catalog_name}""" + // can see the nested ns, with back quote + sql """show tables from `nested`;""" + sql """show tables from `nested.db1`;""" + test { + sql """show tables from nested.db1""" + exception """Unknown catalog 'nested'""" + } + // for "use" stmt, back quote is not necessary + sql """use ${catalog_name}.nested;""" + sql """use ${catalog_name}.`nested.db1`""" + // drop and create nested db1 + sql """drop database if exists `ns1.ns2.ns3` force""" + sql """drop database if exists `ns1.ns2` force""" + sql """drop database if exists `ns1` force""" + qt_sql01 """show databases like 'ns1.ns2.ns3'""" // empty + sql """refresh catalog ${catalog_name}""" + qt_sql02 """show databases like 'ns1.ns2.ns3'""" // empty + + sql """create database `ns1.ns2.ns3`""" + // will see 3 ns, flat + qt_sql03 """show databases like 'ns1'""" // 1 + qt_sql04 """show databases like 'ns1.ns2'""" // 1 + qt_sql05 """show databases like 'ns1.ns2.ns3'""" // 1 + // can create database in each ns + sql """create table ns1.nested_tbl1 (k1 int)"""; + sql """insert into ns1.nested_tbl1 values(101)""" + qt_sql06 """select * from ns1.nested_tbl1""" + + sql """create table `ns1.ns2`.nested_tbl2 (k1 int)"""; + sql """insert into `ns1.ns2`.nested_tbl2 values(102)""" + qt_sql07 """select * from `ns1.ns2`.nested_tbl2""" + + sql """use ${catalog_name}.`ns1.ns2.ns3`""" + sql """create table nested_tbl3 (k1 int)"""; + sql """insert into nested_tbl3 values(103)""" + qt_sql08 """select * from nested_tbl3""" + + // test select column in diff qualified names + qt_sql09 """select ${catalog_name}.ns1.nested_tbl1.k1 from ${catalog_name}.ns1.nested_tbl1""" + qt_sql10 """select ${catalog_name}.`ns1.ns2`.nested_tbl2.k1 from ${catalog_name}.`ns1.ns2`.nested_tbl2""" + sql """use ${catalog_name}.`ns1.ns2`""" + order_qt_sql11 """select ${catalog_name}.ns1.nested_tbl1.k1 from ${catalog_name}.ns1.nested_tbl1 + union all + select k1 from nested_tbl2 + union all + select `ns1.ns2.ns3`.nested_tbl3.k1 from `ns1.ns2.ns3`.nested_tbl3; + """ + // test table exist in each ns + qt_sql12 """show tables from ns1"""; + qt_sql13 """show tables from `ns1.ns2`"""; + qt_sql14 """show tables from `ns1.ns2.ns3`"""; + test { + sql """drop database ns1""" + exception """Namespace ns1 is not empty. 1 tables exist""" + } + test { + sql """drop database `ns1.ns2`""" + exception """Namespace ns1.ns2 is not empty. 1 tables exist""" + } + test { + sql """drop database ${catalog_name}.`ns1.ns2.ns3`""" + exception """Namespace ns1.ns2.ns3 is not empty. 1 tables exist""" + } + // test refresh database and table + sql """refresh database ${catalog_name}.`ns1.ns2`""" + sql """refresh database `ns1.ns2`""" + sql """refresh table ${catalog_name}.`ns1.ns2`.nested_tbl2""" + sql """refresh table `ns1.ns2`.nested_tbl2""" + test { + sql """refresh table `ns1.ns2`.nested_tbl2xxx""" + exception """Table nested_tbl2xxx does not exist in db ns1.ns2""" + } + // drop ns1.ns2 first, we can still see it after refresh, because ns1.ns2.ns3 still exists + sql """drop database `ns1.ns2` force""" + qt_sql15 """show databases like "ns1.ns2"""" // empty + sql """refresh catalog ${catalog_name}""" + qt_sql16 """show databases like "ns1.ns2"""" // 1 + // then we drop ns1.ns2.ns3, after refresh, ns1.ns2 also disappear + sql """drop database `ns1.ns2.ns3` force""" + qt_sql17 """show databases like "ns1.ns2"""" // 1 + qt_sql18 """show databases like "ns1.ns2.ns3"""" // empty + sql """refresh catalog ${catalog_name}""" + qt_sql19 """show databases like "ns1.ns2"""" // empty + qt_sql20 """show databases like "ns1.ns2.ns3"""" // empty + + // recreate ns1.ns2.ns3 + sql """create database `ns1.ns2.ns3`;""" + qt_sql21 """show databases like "ns1.ns2"""" // 1 + qt_sql22 """show databases like "ns1.ns2.ns3"""" // 1 + // drop ns1.ns2.ns3, and ns1.ns2 will disappear too + sql """drop database `ns1.ns2.ns3`""" + sql """refresh catalog ${catalog_name}""" + qt_sql23 """show databases like "ns1.ns2"""" // empty + qt_sql24 """show databases like "ns1.ns2.ns3"""" // empty + + // recreate ns1.ns2.ns3, and create table in ns1.ns2 + sql """create database `ns1.ns2.ns3`;""" + qt_sql25 """show databases like "ns1.ns2"""" // 1 + qt_sql26 """show databases like "ns1.ns2.ns3"""" // 1 + sql """create table `ns1.ns2`.test_table2(k1 int);""" + sql """insert into `ns1.ns2`.test_table2 values(104)""" + qt_sql261 """select * from `ns1.ns2`.test_table2""" + // drop ns1.ns2.ns3, ns1.ns2 will still exist + sql """drop database `ns1.ns2.ns3`""" + sql """refresh catalog ${catalog_name}""" + qt_sql27 """show databases like "ns1.ns2"""" // 1 + qt_sql28 """show databases like "ns1.ns2.ns3"""" // empty + qt_sql29 """select * from `ns1.ns2`.test_table2""" + // drop `ns1.ns2`.test_table2, and then ns1.ns2 will disappeal + sql """drop table `ns1.ns2`.test_table2""" + sql """refresh catalog ${catalog_name}""" + qt_sql30 """show databases like "ns1.ns2"""" // empty + + // test dropping and creating table in nested ns spark created + sql """drop table if exists `nested.db1`.spark_table""" + sql """create table `nested.db1`.spark_table (k1 int)""" + sql """insert into `nested.db1`.spark_table values(105)""" + qt_sql31 """select * from `nested.db1`.spark_table""" + + + // 4. + // iceberg.rest.nested-namespace-enabled = false + // set global enable_nested_namespace = true + sql """set global enable_nested_namespace = true""" + sql """alter catalog ${catalog_name} set properties("iceberg.rest.nested-namespace-enabled" = "false");""" + sql """switch ${catalog_name}""" + // can not see the nested ns + qt_sql32 """show databases like "nested.db1";""" // empty + test { + sql """use ${catalog_name}.`nested.db1`""" + exception """Unknown database 'nested.db1'""" + } + + // can create nested ns, but can not drop because nested ns can not be seen + test { + sql """drop database `nested.db1`""" + exception """Can't drop database 'nested.db1'; database doesn't exist""" + } + sql """create database if not exists `nsa.nsb`""" + sql """create database if not exists `nsa.nsb.nsc`""" + // can only see nsa + qt_sql33 """show databases like "nsa"""" // 1 + qt_sql34 """show databases like "nsa.nsb"""" // empty + // can create and drop table in nsa + sql """drop table if exists nsa.nsa_tbl1""" + sql """create table nsa.nsa_tbl1 (k1 int);""" + sql """insert into nsa.nsa_tbl1 values(106)""" + qt_sql35 """select * from nsa.nsa_tbl1""" + sql """drop table nsa.nsa_tbl1""" + test { + sql """select * from nsa.nsa_tbl1""" + exception """Table [nsa_tbl1] does not exist in database [nsa]""" + } + + // 5. test internal + sql """switch internal""" + sql """set global enable_nested_namespace = true""" + // create nested ns + sql """drop database if exists `idb1.idb2.idb3`""" + sql """drop database if exists `idb1.idb2`""" + sql """drop database if exists `idb1`""" + sql """create database `idb1`""" + sql """create database `idb1.idb2`""" + sql """create database `idb1.idb2.idb3`""" + qt_sql1001 """show databases like "idb1"""" + qt_sql1002 """show databases like "idb1.idb2"""" + qt_sql1003 """show databases like "idb1.idb2.idb3"""" + + // create table + sql """create table idb1.itbl1 (k1 int) properties("replication_num" = "1")""" + sql """create table `idb1.idb2`.itbl2 (k1 int) properties("replication_num" = "1")""" + sql """use internal.`idb1.idb2.idb3`""" + sql """create table itbl3 (k1 int) properties("replication_num" = "1")""" + + // insert + sql """insert into idb1.itbl1 values(201)""" + sql """insert into `idb1.idb2`.itbl2 values(202)""" + sql """use internal.`idb1.idb2.idb3`""" + sql """insert into itbl3 values(203)""" + + // query + qt_sql101 """select * from idb1.itbl1""" + qt_sql103 """select `idb1.idb2`.itbl2.k1 from `idb1.idb2`.itbl2""" + sql """use internal.`idb1.idb2.idb3`""" + order_qt_sql104 """select `idb1.idb2`.itbl2.k1 from `idb1.idb2`.itbl2 + union all + select idb1.itbl1.k1 from idb1.itbl1 + union all + select itbl3.k1 from itbl3 + """ + // disable + sql """set global enable_nested_namespace = false""" + // still can see nested ns + qt_sql2001 """show databases like "idb1"""" + qt_sql2002 """show databases like "idb1.idb2"""" + qt_sql2003 """show databases like "idb1.idb2.idb3"""" + + sql """ unset global variable enable_nested_namespace;""" +}