diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index bc1f8576ca5446..7d7fc29b4df4a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -17,13 +17,9 @@ package org.apache.doris.catalog; -import org.apache.doris.analysis.RefreshCatalogStmt; -import org.apache.doris.analysis.RefreshDbStmt; -import org.apache.doris.analysis.RefreshTableStmt; import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.CatalogFactory; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.CatalogLog; import org.apache.doris.datasource.ExternalCatalog; @@ -35,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; import org.apache.doris.persist.OperationType; +import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -57,10 +54,10 @@ public class RefreshManager { private Map refreshMap = Maps.newConcurrentMap(); // Refresh catalog - public void handleRefreshCatalog(RefreshCatalogStmt stmt) throws UserException { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(stmt.getCatalogName()); - CatalogLog log = CatalogFactory.createCatalogLog(catalog.getId(), stmt); - refreshCatalogInternal(catalog, log.isInvalidCache()); + public void handleRefreshCatalog(String catalogName, boolean invalidCache) throws UserException { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalogName); + refreshCatalogInternal(catalog, invalidCache); + CatalogLog log = CatalogLog.createForRefreshCatalog(catalog.getId(), invalidCache); Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_REFRESH_CATALOG, log); } @@ -82,9 +79,7 @@ private void refreshCatalogInternal(CatalogIf catalog, boolean invalidCache) { } // Refresh database - public void handleRefreshDb(RefreshDbStmt stmt) throws DdlException { - String catalogName = stmt.getCatalogName(); - String dbName = stmt.getDbName(); + public void handleRefreshDb(String catalogName, String dbName, boolean invalidCache) throws DdlException { Env env = Env.getCurrentEnv(); CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog(); if (catalog == null) { @@ -94,40 +89,42 @@ public void handleRefreshDb(RefreshDbStmt stmt) throws DdlException { throw new DdlException("Only support refresh database in external catalog"); } DatabaseIf db = catalog.getDbOrDdlException(dbName); - ((ExternalDatabase) db).setUnInitialized(stmt.isInvalidCache()); + refreshDbInternal((ExternalDatabase) db, invalidCache); - ExternalObjectLog log = new ExternalObjectLog(); - log.setCatalogId(catalog.getId()); - log.setDbId(db.getId()); - log.setInvalidCache(stmt.isInvalidCache()); + ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName(), invalidCache); Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log); } public void replayRefreshDb(ExternalObjectLog log) { - refreshDbInternal(log.getCatalogId(), log.getDbId(), log.isInvalidCache()); - } + ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(log.getCatalogId()); + if (catalog == null) { + LOG.warn("failed to find catalog when replaying refresh db: {}", log.debugForRefreshDb()); + } + Optional> db; + if (!Strings.isNullOrEmpty(log.getDbName())) { + db = catalog.getDbForReplay(log.getDbName()); + } else { + db = catalog.getDbForReplay(log.getDbId()); + } - private void refreshDbInternal(long catalogId, long dbId, boolean invalidCache) { - ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId); - Optional> db = catalog.getDbForReplay(dbId); - // Database may not exist if 'use_meta_cache' is true. - // Because each FE fetch the meta data independently. - db.ifPresent(e -> { - e.setUnInitialized(invalidCache); - LOG.info("refresh database {} in catalog {} with invalidCache {}", e.getFullName(), - catalog.getName(), invalidCache); - }); + if (!db.isPresent()) { + LOG.warn("failed to find db when replaying refresh db: {}", log.debugForRefreshDb()); + } else { + refreshDbInternal(db.get(), log.isInvalidCache()); + } } - // Refresh table - public void handleRefreshTable(RefreshTableStmt stmt) throws UserException { - String catalogName = stmt.getCtl(); - String dbName = stmt.getDbName(); - String tableName = stmt.getTblName(); - refreshTable(catalogName, dbName, tableName, false); + private void refreshDbInternal(ExternalDatabase db, boolean invalidCache) { + db.setUnInitialized(); + if (invalidCache) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(db.getCatalog().getId(), db.getFullName()); + } + LOG.info("refresh database {} in catalog {} with invalidCache {}", db.getFullName(), + db.getCatalog().getName(), invalidCache); } - public void refreshTable(String catalogName, String dbName, String tableName, boolean ignoreIfNotExists) + // Refresh table + public void handleRefreshTable(String catalogName, String dbName, String tableName, boolean ignoreIfNotExists) throws DdlException { Env env = Env.getCurrentEnv(); CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog(); @@ -153,33 +150,41 @@ public void refreshTable(String catalogName, String dbName, String tableName, bo } return; } - refreshTableInternal(catalog, db, table, 0); + refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, 0); - ExternalObjectLog log = new ExternalObjectLog(); - log.setCatalogId(catalog.getId()); - log.setDbId(db.getId()); - log.setTableId(table.getId()); + ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(catalog.getId(), db.getFullName(), + table.getName()); Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); } public void replayRefreshTable(ExternalObjectLog log) { ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(log.getCatalogId()); if (catalog == null) { - LOG.warn("failed to find catalog replaying refresh table {}", log.getCatalogId()); + LOG.warn("failed to find catalog when replaying refresh table: {}", log.debugForRefreshTable()); return; } - Optional> db = catalog.getDbForReplay(log.getDbId()); + Optional> db; + if (!Strings.isNullOrEmpty(log.getDbName())) { + db = catalog.getDbForReplay(log.getDbName()); + } else { + db = catalog.getDbForReplay(log.getDbId()); + } // See comment in refreshDbInternal for why db and table may be null. if (!db.isPresent()) { - LOG.warn("failed to find db replaying refresh table {}", log.getDbId()); + LOG.warn("failed to find db when replaying refresh table: {}", log.debugForRefreshTable()); return; } - Optional table = db.get().getTableForReplay(log.getTableId()); + Optional table; + if (!Strings.isNullOrEmpty(log.getTableName())) { + table = db.get().getTableForReplay(log.getTableName()); + } else { + table = db.get().getTableForReplay(log.getTableId()); + } if (!table.isPresent()) { - LOG.warn("failed to find table replaying refresh table {}", log.getTableId()); + LOG.warn("failed to find table when replaying refresh table: {}", log.debugForRefreshTable()); return; } - refreshTableInternal(catalog, db.get(), table.get(), log.getLastUpdateTime()); + refreshTableInternal(db.get(), table.get(), log.getLastUpdateTime()); } public void refreshExternalTableFromEvent(String catalogName, String dbName, String tableName, @@ -200,18 +205,17 @@ public void refreshExternalTableFromEvent(String catalogName, String dbName, Str if (table == null) { return; } - refreshTableInternal(catalog, db, table, updateTime); + refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime); } - public void refreshTableInternal(CatalogIf catalog, DatabaseIf db, TableIf table, long updateTime) { - if (table instanceof ExternalTable) { - ((ExternalTable) table).unsetObjectCreated(); - } - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache((ExternalTable) table); + public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime) { + table.unsetObjectCreated(); if (table instanceof HMSExternalTable && updateTime > 0) { ((HMSExternalTable) table).setEventUpdateTime(updateTime); } - LOG.info("refresh table {} from db {} in catalog {}", table.getName(), db.getFullName(), catalog.getName()); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table); + LOG.info("refresh table {}, id {} from db {} in catalog {}", + table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName()); } // Refresh partition diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java index 558d8b9ae10966..f7f1d5047a62e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java @@ -18,9 +18,6 @@ package org.apache.doris.datasource; import org.apache.doris.analysis.CreateCatalogStmt; -import org.apache.doris.analysis.DropCatalogStmt; -import org.apache.doris.analysis.RefreshCatalogStmt; -import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Resource; import org.apache.doris.common.DdlException; @@ -48,22 +45,6 @@ public class CatalogFactory { private static final Logger LOG = LogManager.getLogger(CatalogFactory.class); - /** - * Convert the sql statement into catalog log. - */ - public static CatalogLog createCatalogLog(long catalogId, StatementBase stmt) { - CatalogLog log = new CatalogLog(); - if (stmt instanceof DropCatalogStmt) { - log.setCatalogId(catalogId); - } else if (stmt instanceof RefreshCatalogStmt) { - log.setCatalogId(catalogId); - log.setInvalidCache(((RefreshCatalogStmt) stmt).isInvalidCache()); - } else { - throw new RuntimeException("Unknown stmt for catalog manager " + stmt.getClass().getSimpleName()); - } - return log; - } - /** * create the catalog instance from catalog log. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogLog.java index d583fcbb64e63c..d862a3ef44cfe1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogLog.java @@ -62,6 +62,13 @@ public class CatalogLog implements Writable { @SerializedName(value = "comment") private String comment; + public static CatalogLog createForRefreshCatalog(long catalogId, boolean invalidCache) { + CatalogLog log = new CatalogLog(); + log.setCatalogId(catalogId); + log.setInvalidCache(invalidCache); + return log; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index c7ad269844b316..9ed890f08c0058 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -158,7 +158,6 @@ public abstract class ExternalCatalog // db name does not contains "default_cluster" protected Map dbNameToId = Maps.newConcurrentMap(); private boolean objectCreated = false; - protected boolean invalidCacheInInit = true; protected ExternalMetadataOps metadataOps; protected TransactionManager transactionManager; @@ -354,7 +353,7 @@ private void buildMetaCache() { localDbName -> Optional.ofNullable( buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, true)), - (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); + (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized())); } } @@ -580,11 +579,10 @@ private void refreshOnlyCatalogCache(boolean invalidCache) { } else if (!useMetaCache.get()) { this.initialized = false; for (ExternalDatabase db : idToDb.values()) { - db.setUnInitialized(invalidCache); + db.setUnInitialized(); } } } - this.invalidCacheInInit = invalidCache; if (invalidCache) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalogCache(id); } @@ -1103,6 +1101,7 @@ public boolean createTable(CreateTableStmt stmt) throws UserException { // we should get the table stored in Doris, and use local name in edit log. CreateTableInfo info = new CreateTableInfo(getName(), stmt.getDbName(), stmt.getTableName()); Env.getCurrentEnv().getEditLog().logCreateTable(info); + LOG.info("finished to create table {}.{}.{}", getName(), stmt.getDbName(), stmt.getTableName()); } return res; } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 9b151a2f354802..7141a484a452eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -94,7 +94,6 @@ public abstract class ExternalDatabase protected long lastUpdateTime; protected final InitDatabaseLog.Type dbLogType; protected ExternalCatalog extCatalog; - protected boolean invalidCacheInInit = true; private MetaCache metaCache; @@ -131,9 +130,9 @@ public void setTableExtCatalog(ExternalCatalog extCatalog) { } } - public synchronized void setUnInitialized(boolean invalidCache) { + public synchronized void setUnInitialized() { this.initialized = false; - this.invalidCacheInInit = invalidCache; + this.lowerCaseToTableName = Maps.newConcurrentMap(); if (extCatalog.getUseMetaCache().isPresent()) { if (extCatalog.getUseMetaCache().get() && metaCache != null) { metaCache.invalidateAll(); @@ -143,9 +142,6 @@ public synchronized void setUnInitialized(boolean invalidCache) { } } } - if (invalidCache) { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(extCatalog.getId(), name); - } } public boolean isInitialized() { @@ -315,6 +311,7 @@ private void buildMetaCache() { private List> listTableNames() { List> tableNames; + lowerCaseToTableName.clear(); if (name.equals(InfoSchemaDb.DATABASE_NAME)) { tableNames = ExternalInfoSchemaDatabase.listTableNames().stream() .map(tableName -> { @@ -339,6 +336,10 @@ private List> listTableNames() { return Pair.of(tableName, localTableName); }).collect(Collectors.toList()); } + if (LOG.isDebugEnabled()) { + LOG.debug("after list tables in {}.{}, lowerCaseToTableName: {}", + getCatalog().getName(), getFullName(), lowerCaseToTableName); + } // Check for conflicts when stored table names or meta names are case-insensitive if (Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) || this.isStoredTableNamesLowerCase() @@ -610,37 +611,44 @@ public Set getTableNamesWithLock() { @Override public T getTableNullable(String tableName) { makeSureInitialized(); + String finalName = tableName; if (this.isStoredTableNamesLowerCase()) { - tableName = tableName.toLowerCase(); + finalName = tableName.toLowerCase(); } if (this.isTableNamesCaseInsensitive()) { - String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (realTableName == null) { + finalName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (finalName == null) { // Here we need to execute listTableNames() once to fill in lowerCaseToTableName // to prevent lowerCaseToTableName from being empty in some cases listTableNames(); - tableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (tableName == null) { + finalName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (finalName == null) { + if (LOG.isDebugEnabled()) { + LOG.info("failed to get final table name from: {}.{}.{}", + getCatalog().getName(), getFullName(), tableName); + } return null; } - } else { - tableName = realTableName; } } if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true") && (this.isTableNamesCaseInsensitive())) { - tableName = tableName.toLowerCase(); + finalName = tableName.toLowerCase(); + } + if (LOG.isDebugEnabled()) { + LOG.info("get table {} from database: {}.{}, final name is: {}", + tableName, getCatalog().getName(), getFullName(), finalName); } if (extCatalog.getUseMetaCache().get()) { // must use full qualified name to generate id. // otherwise, if 2 databases have the same table name, the id will be the same. - return metaCache.getMetaObj(tableName, - Util.genIdByName(extCatalog.getName(), name, tableName)).orElse(null); + return metaCache.getMetaObj(finalName, + Util.genIdByName(extCatalog.getName(), name, finalName)).orElse(null); } else { - if (!tableNameToId.containsKey(tableName)) { + if (!tableNameToId.containsKey(finalName)) { return null; } - return idToTbl.get(tableNameToId.get(tableName)); + return idToTbl.get(tableNameToId.get(finalName)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java index 7729ede264afdc..04d90c9a049d24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java @@ -21,17 +21,16 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; +import com.google.common.base.Strings; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.Getter; -import lombok.NoArgsConstructor; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.List; -@NoArgsConstructor @Getter @Data public class ExternalObjectLog implements Writable { @@ -59,6 +58,26 @@ public class ExternalObjectLog implements Writable { @SerializedName(value = "lastUpdateTime") private long lastUpdateTime; + private ExternalObjectLog() { + + } + + public static ExternalObjectLog createForRefreshDb(long catalogId, String dbName, boolean invalidCache) { + ExternalObjectLog externalObjectLog = new ExternalObjectLog(); + externalObjectLog.setCatalogId(catalogId); + externalObjectLog.setDbName(dbName); + externalObjectLog.setInvalidCache(invalidCache); + return externalObjectLog; + } + + public static ExternalObjectLog createForRefreshTable(long catalogId, String dbName, String tblName) { + ExternalObjectLog externalObjectLog = new ExternalObjectLog(); + externalObjectLog.setCatalogId(catalogId); + externalObjectLog.setDbName(dbName); + externalObjectLog.setTableName(tblName); + return externalObjectLog; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); @@ -68,4 +87,31 @@ public static ExternalObjectLog read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, ExternalObjectLog.class); } + + public String debugForRefreshDb() { + StringBuilder sb = new StringBuilder(); + sb.append("[catalogId: " + catalogId + ", "); + if (!Strings.isNullOrEmpty(dbName)) { + sb.append("dbName: " + dbName + "]"); + } else { + sb.append("dbId: " + dbId + "]"); + } + return sb.toString(); + } + + public String debugForRefreshTable() { + StringBuilder sb = new StringBuilder(); + sb.append("[catalogId: " + catalogId + ", "); + if (!Strings.isNullOrEmpty(dbName)) { + sb.append("dbName: " + dbName + ", "); + } else { + sb.append("dbId: " + dbId + ", "); + } + if (!Strings.isNullOrEmpty(tableName)) { + sb.append("tableName: " + tableName + "]"); + } else { + sb.append("tableId: " + tableId + "]"); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index 3d79bc9c31efb2..1cee2899e592e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -289,8 +289,10 @@ In hive, this table only can insert not update(not report error,but not actually public void afterCreateTable(String dbName, String tblName) { Optional> db = catalog.getDbForReplay(dbName); if (db.isPresent()) { - db.get().setUnInitialized(true); + db.get().setUnInitialized(); } + LOG.info("after create table {}.{}.{}, is db exists: {}", + getCatalog().getName(), dbName, tblName, db.isPresent()); } @Override @@ -319,8 +321,14 @@ public void dropTableImpl(ExternalTable dorisTable, boolean ifExists) throws Ddl public void afterDropTable(String dbName, String tblName) { Optional> db = catalog.getDbForReplay(dbName); if (db.isPresent()) { - db.get().setUnInitialized(true); + Optional table = db.get().getTableForReplay(tblName); + if (table.isPresent()) { + Env.getCurrentEnv().getRefreshManager().refreshTableInternal(db.get(), (ExternalTable) table.get(), 0); + } + db.get().setUnInitialized(); } + LOG.info("after drop table {}.{}.{}, is db exists: {}", + getCatalog().getName(), dbName, tblName, db.isPresent()); } @Override @@ -344,7 +352,7 @@ public void afterTruncateTable(String dbName, String tblName) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache((ExternalTable) dorisTable.get()); } db.get().setLastUpdateTime(System.currentTimeMillis()); - db.get().setUnInitialized(true); + db.get().setUnInitialized(); } } catch (Exception e) { LOG.warn("exception when calling afterTruncateTable for db: {}, table: {}, error: {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 1fcc7083258910..ca57ab1d54332d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; -import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; @@ -311,8 +310,10 @@ public boolean performCreateTable(CreateTableStmt stmt) throws UserException { public void afterCreateTable(String dbName, String tblName) { Optional> db = dorisCatalog.getDbForReplay(dbName); if (db.isPresent()) { - db.get().setUnInitialized(true); + db.get().setUnInitialized(); } + LOG.info("after create table {}.{}.{}, is db exists: {}", + dorisCatalog.getName(), dbName, tblName, db.isPresent()); } @Override @@ -337,9 +338,13 @@ public void dropTableImpl(ExternalTable dorisTable, boolean ifExists) throws Ddl public void afterDropTable(String dbName, String tblName) { Optional> db = dorisCatalog.getDbForReplay(dbName); if (db.isPresent()) { - db.get().setUnInitialized(true); + Optional table = db.get().getTableForReplay(tblName); + if (table.isPresent()) { + Env.getCurrentEnv().getRefreshManager().refreshTableInternal(db.get(), (ExternalTable) table.get(), 0); + } + db.get().setUnInitialized(); } - LOG.info("after drop table {}.{}.{}. is db present: {}", + LOG.info("after drop table {}.{}.{}. is db exists: {}", dorisCatalog.getName(), dbName, tblName, db.isPresent()); } @@ -423,7 +428,7 @@ public void afterOperateOnBranchOrTag(String dbName, String tblName) { Optional tbl = db.get().getTableForReplay(tblName); if (tbl.isPresent()) { Env.getCurrentEnv().getRefreshManager() - .refreshTableInternal(dorisCatalog, db.get(), (TableIf) tbl.get(), + .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), System.currentTimeMillis()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java index ab7b6d3e36170a..9b8899e0940605 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java @@ -95,7 +95,7 @@ protected void onComplete() throws UserException { transactionManager.commit(txnId); summaryProfile.ifPresent(SummaryProfile::setTransactionEndTime); txnStatus = TransactionStatus.COMMITTED; - Env.getCurrentEnv().getRefreshManager().refreshTable( + Env.getCurrentEnv().getRefreshManager().handleRefreshTable( catalogName, table.getDatabase().getFullName(), table.getName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshCatalogCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshCatalogCommand.java index 7b6433038197dd..7b22fcffc8badf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshCatalogCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshCatalogCommand.java @@ -99,7 +99,7 @@ public void handleRefreshCatalog() throws AnalysisException { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { validate(); - handleRefreshCatalog(); + Env.getCurrentEnv().getRefreshManager().handleRefreshCatalog(catalogName, invalidCache); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java index 010adbb05d1cba..ca3be8294f99e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java @@ -18,18 +18,12 @@ package org.apache.doris.nereids.trees.plans.commands.refresh; import org.apache.doris.analysis.StmtType; -import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.MysqlDb; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.datasource.ExternalCatalog; -import org.apache.doris.datasource.ExternalDatabase; -import org.apache.doris.datasource.ExternalObjectLog; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.Command; @@ -96,32 +90,10 @@ private void validate(ConnectContext ctx) throws AnalysisException { invalidCache = invalidConfig == null || invalidConfig.equalsIgnoreCase("true"); } - /** - * Refresh database - */ - public void handleRefreshDb() throws DdlException { - Env env = Env.getCurrentEnv(); - CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog(); - if (catalog == null) { - throw new DdlException("Catalog " + catalogName + " doesn't exist."); - } - if (!(catalog instanceof ExternalCatalog)) { - throw new DdlException("Only support refresh database in external catalog"); - } - DatabaseIf db = catalog.getDbOrDdlException(dbName); - ((ExternalDatabase) db).setUnInitialized(invalidCache); - - ExternalObjectLog log = new ExternalObjectLog(); - log.setCatalogId(catalog.getId()); - log.setDbId(db.getId()); - log.setInvalidCache(invalidCache); - Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log); - } - @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { validate(ctx); - handleRefreshDb(); + Env.getCurrentEnv().getRefreshManager().handleRefreshDb(catalogName, dbName, invalidCache); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshTableCommand.java index 71d77ab22b36fd..9a8c19530568fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshTableCommand.java @@ -52,7 +52,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { checkRefreshTableAccess(); // refresh table execute logic Env.getCurrentEnv().getRefreshManager() - .refreshTable(tableNameInfo.getCtl(), tableNameInfo.getDb(), tableNameInfo.getTbl(), false); + .handleRefreshTable(tableNameInfo.getCtl(), tableNameInfo.getDb(), tableNameInfo.getTbl(), false); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 96893b5a8e050a..c8a60497c78760 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -235,9 +235,13 @@ public static void execute(Env env, DdlStmt ddlStmt) throws Exception { } else if (ddlStmt instanceof AlterDatabasePropertyStmt) { env.alterDatabaseProperty((AlterDatabasePropertyStmt) ddlStmt); } else if (ddlStmt instanceof RefreshTableStmt) { - env.getRefreshManager().handleRefreshTable((RefreshTableStmt) ddlStmt); + RefreshTableStmt refreshTableStmt = (RefreshTableStmt) ddlStmt; + env.getRefreshManager().handleRefreshTable(refreshTableStmt.getCtl(), refreshTableStmt.getDbName(), + refreshTableStmt.getTblName(), false); } else if (ddlStmt instanceof RefreshDbStmt) { - env.getRefreshManager().handleRefreshDb((RefreshDbStmt) ddlStmt); + RefreshDbStmt refreshDbStmt = (RefreshDbStmt) ddlStmt; + env.getRefreshManager().handleRefreshDb(refreshDbStmt.getCatalogName(), refreshDbStmt.getDbName(), + refreshDbStmt.isInvalidCache()); } else if (ddlStmt instanceof AlterColocateGroupStmt) { env.getColocateTableIndex().alterColocateGroup((AlterColocateGroupStmt) ddlStmt); } else if (ddlStmt instanceof AlterWorkloadGroupStmt) { @@ -255,7 +259,9 @@ public static void execute(Env env, DdlStmt ddlStmt) throws Exception { } else if (ddlStmt instanceof DropMaterializedViewStmt) { env.dropMaterializedView((DropMaterializedViewStmt) ddlStmt); } else if (ddlStmt instanceof RefreshCatalogStmt) { - env.getRefreshManager().handleRefreshCatalog((RefreshCatalogStmt) ddlStmt); + RefreshCatalogStmt refreshCatalogStmt = (RefreshCatalogStmt) ddlStmt; + env.getRefreshManager() + .handleRefreshCatalog(refreshCatalogStmt.getCatalogName(), refreshCatalogStmt.isInvalidCache()); } else if (ddlStmt instanceof RefreshLdapStmt) { env.getAuth().refreshLdap((RefreshLdapStmt) ddlStmt); } else if (ddlStmt instanceof CleanProfileStmt) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java index b61199fa0bae3f..a62f90cbaea7bf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java @@ -111,7 +111,7 @@ public void setUp() throws IOException { RefreshManager refreshManager = Mockito.mock(RefreshManager.class); Mockito.when(mockEnv.getRefreshManager()).thenReturn(refreshManager); Mockito.doNothing().when(refreshManager) - .refreshTableInternal(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong()); + .refreshTableInternal(Mockito.any(), Mockito.any(), Mockito.anyLong()); } @AfterEach