Skip to content

branch-3.1: [fix](hive) use the remote name when get meta from hive system. #52561 #52817 #52881 #52989 #53030

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 10 commits into from
Jul 11, 2025
Merged

branch-3.1: [fix](hive) use the remote name when get meta from hive system. #52561 #52817 #52881 #52989 #53030

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c88321f
fe code
morningman Jul 10, 2025
f275e44
add regression test
morningman Jul 10, 2025
9983787
[fix](paimon) fix CacheException msg in loadSnapshot (#52881)
suxiaogang223 Jul 8, 2025
599290e
[fix](test)(jdbc) fix some external regression test cases and jdbc bu…
morningman Jul 6, 2025
6cf58cf
fix format
morningman Jul 10, 2025
4796704
fix ut
morningman Jul 10, 2025
9870b89
fix test
morningman Jul 10, 2025
d862001
1
morningman Jul 10, 2025
83badb6
[fix](catalog) fix refresh logic #52989
morningman Jul 9, 2025
b158df1
fix compile
morningman Jul 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,19 +346,19 @@ private void processAlterTableForExternalTable(
setExternalTableAutoAnalyzePolicy(table, alterClauses);
} else if (alterClause instanceof CreateOrReplaceBranchClause) {
table.getCatalog().createOrReplaceBranch(
table.getDbName(), table.getName(),
table,
((CreateOrReplaceBranchClause) alterClause).getBranchInfo());
} else if (alterClause instanceof CreateOrReplaceTagClause) {
table.getCatalog().createOrReplaceTag(
table.getDbName(), table.getName(),
table,
((CreateOrReplaceTagClause) alterClause).getTagInfo());
} else if (alterClause instanceof DropBranchClause) {
table.getCatalog().dropBranch(
table.getDbName(), table.getName(),
table,
((DropBranchClause) alterClause).getDropBranchInfo());
} else if (alterClause instanceof DropTagClause) {
table.getCatalog().dropTag(
table.getDbName(), table.getName(),
table,
((DropTagClause) alterClause).getDropTagInfo());
} else {
throw new UserException("Invalid alter operations for external table: " + alterClauses);
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,7 @@ public boolean registerTable(TableIf table) {
return result;
}

@Override
public void unregisterTable(String tableName) {
if (Env.isStoredTableNamesLowerCase()) {
tableName = tableName.toLowerCase();
Expand Down
19 changes: 10 additions & 9 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@
import org.apache.doris.analysis.SetType;
import org.apache.doris.analysis.ShowAlterStmt.AlterType;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.TableRef;
import org.apache.doris.analysis.TableRenameClause;
import org.apache.doris.analysis.TruncateTableStmt;
import org.apache.doris.analysis.UninstallPluginStmt;
Expand Down Expand Up @@ -3316,7 +3318,7 @@ public void createDb(CreateDbStmt stmt) throws DdlException {
} else {
catalogIf = catalogMgr.getCatalog(stmt.getCtlName());
}
catalogIf.createDb(stmt);
catalogIf.createDb(stmt.getFullDbName(), stmt.isSetIfNotExists(), stmt.getProperties());
}

// For replay edit log, need't lock metadata
Expand All @@ -3342,7 +3344,7 @@ public void dropDb(DropDbStmt stmt) throws DdlException {
} else {
catalogIf = catalogMgr.getCatalog(stmt.getCtlName());
}
catalogIf.dropDb(stmt);
catalogIf.dropDb(stmt.getDbName(), stmt.isSetIfExists(), stmt.isForceDrop());
}

public void replayDropDb(DropDbInfo info) throws DdlException {
Expand Down Expand Up @@ -4294,12 +4296,8 @@ public void replayAlterExternalTableSchema(String dbName, String tableName, List
public void dropTable(DropTableStmt stmt) throws DdlException {
CatalogIf<?> catalogIf = catalogMgr.getCatalogOrException(stmt.getCatalogName(),
catalog -> new DdlException(("Unknown catalog " + catalog)));
catalogIf.dropTable(stmt);
}

public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, boolean isReplay,
Long recycleTime) {
return getInternalCatalog().unprotectDropTable(db, table, isForceDrop, isReplay, recycleTime);
catalogIf.dropTable(stmt.getDbName(), stmt.getTableName(), stmt.isView(), stmt.isMaterializedView(),
stmt.isSetIfExists(), stmt.isForceDrop());
}

public void replayDropTable(Database db, long tableId, boolean isForceDrop,
Expand Down Expand Up @@ -5999,7 +5997,10 @@ public String dumpImage() {
public void truncateTable(TruncateTableStmt stmt) throws DdlException {
CatalogIf<?> catalogIf = catalogMgr.getCatalogOrException(stmt.getTblRef().getName().getCtl(),
catalog -> new DdlException(("Unknown catalog " + catalog)));
catalogIf.truncateTable(stmt);
TableRef tableRef = stmt.getTblRef();
TableName tableName = tableRef.getName();
catalogIf.truncateTable(tableName.getDb(), tableName.getTbl(), tableRef.getPartitionNames(),
false, stmt.toSqlWithoutTable());
}

public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException {
Expand Down
5 changes: 2 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DbName;
import org.apache.doris.analysis.DistributionDesc;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.HashDistributionDesc;
import org.apache.doris.analysis.KeysDesc;
import org.apache.doris.analysis.ModifyColumnClause;
Expand Down Expand Up @@ -348,8 +347,8 @@ private boolean created() {
if (!optionalColumn.isPresent() || !optionalColumn.get().isAllowNull()) {
try {
Env.getCurrentEnv().getInternalCatalog()
.dropTable(new DropTableStmt(true, new TableName(null,
StatisticConstants.DB_NAME, StatisticConstants.TABLE_STATISTIC_TBL_NAME), true));
.dropTable(StatisticConstants.DB_NAME, StatisticConstants.TABLE_STATISTIC_TBL_NAME,
false, false, true, true);
} catch (Exception e) {
LOG.warn("Failed to drop outdated table", e);
}
Expand Down
15 changes: 13 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ public class JdbcTable extends Table {
private boolean connectionPoolKeepAlive;

private ExternalFunctionRules functionRules;
// This is used for edit log
@SerializedName("frs")
private String functionRulesString;

static {
Map<String, TOdbcTableType> tempMap = new CaseInsensitiveMap();
Expand Down Expand Up @@ -419,8 +422,9 @@ private void validate(Map<String, String> properties) throws DdlException {

private void checkAndSetExternalFunctionRules(Map<String, String> properties) throws DdlException {
ExternalFunctionRules.check(properties.getOrDefault(JdbcResource.FUNCTION_RULES, ""));
this.functionRules = ExternalFunctionRules.create(jdbcTypeName,
properties.getOrDefault(JdbcResource.FUNCTION_RULES, ""));
String functionRulesString = properties.getOrDefault(JdbcResource.FUNCTION_RULES, "");
this.functionRules = ExternalFunctionRules.create(jdbcTypeName, functionRulesString);
this.functionRulesString = functionRulesString;
}

/**
Expand Down Expand Up @@ -529,4 +533,11 @@ public void setExternalFunctionRules(ExternalFunctionRules functionRules) {
public ExternalFunctionRules getExternalFunctionRules() {
return functionRules;
}

@Override
public void gsonPostProcess() throws IOException {
super.gsonPostProcess();
functionRules = ExternalFunctionRules.create(jdbcTypeName, Strings.nullToEmpty(functionRulesString));
}
}

113 changes: 58 additions & 55 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,9 @@
package org.apache.doris.catalog;

import org.apache.doris.analysis.RefreshCatalogStmt;
import org.apache.doris.analysis.RefreshDbStmt;
import org.apache.doris.analysis.RefreshTableStmt;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogFactory;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.CatalogLog;
import org.apache.doris.datasource.ExternalCatalog;
Expand All @@ -35,6 +32,7 @@
import org.apache.doris.persist.OperationType;
import org.apache.doris.qe.DdlExecutor;

import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -57,10 +55,10 @@ public class RefreshManager {
private Map<Long, Integer[]> refreshMap = Maps.newConcurrentMap();

// Refresh catalog
public void handleRefreshCatalog(RefreshCatalogStmt stmt) throws UserException {
CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(stmt.getCatalogName());
CatalogLog log = CatalogFactory.createCatalogLog(catalog.getId(), stmt);
refreshCatalogInternal(catalog, log.isInvalidCache());
public void handleRefreshCatalog(String catalogName, boolean invalidCache) throws UserException {
CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalogName);
refreshCatalogInternal(catalog, invalidCache);
CatalogLog log = CatalogLog.createForRefreshCatalog(catalog.getId(), invalidCache);
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_REFRESH_CATALOG, log);
}

Expand All @@ -82,9 +80,7 @@ private void refreshCatalogInternal(CatalogIf catalog, boolean invalidCache) {
}

// Refresh database
public void handleRefreshDb(RefreshDbStmt stmt) throws DdlException {
String catalogName = stmt.getCatalogName();
String dbName = stmt.getDbName();
public void handleRefreshDb(String catalogName, String dbName, boolean invalidCache) throws DdlException {
Env env = Env.getCurrentEnv();
CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
if (catalog == null) {
Expand All @@ -94,40 +90,42 @@ public void handleRefreshDb(RefreshDbStmt stmt) throws DdlException {
throw new DdlException("Only support refresh database in external catalog");
}
DatabaseIf db = catalog.getDbOrDdlException(dbName);
((ExternalDatabase) db).setUnInitialized(stmt.isInvalidCache());
refreshDbInternal((ExternalDatabase) db, invalidCache);

ExternalObjectLog log = new ExternalObjectLog();
log.setCatalogId(catalog.getId());
log.setDbId(db.getId());
log.setInvalidCache(stmt.isInvalidCache());
ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName(), invalidCache);
Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log);
}

public void replayRefreshDb(ExternalObjectLog log) {
refreshDbInternal(log.getCatalogId(), log.getDbId(), log.isInvalidCache());
}
ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(log.getCatalogId());
if (catalog == null) {
LOG.warn("failed to find catalog when replaying refresh db: {}", log.debugForRefreshDb());
}
Optional<ExternalDatabase<? extends ExternalTable>> db;
if (!Strings.isNullOrEmpty(log.getDbName())) {
db = catalog.getDbForReplay(log.getDbName());
} else {
db = catalog.getDbForReplay(log.getDbId());
}

private void refreshDbInternal(long catalogId, long dbId, boolean invalidCache) {
ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId);
Optional<ExternalDatabase<? extends ExternalTable>> db = catalog.getDbForReplay(dbId);
// Database may not exist if 'use_meta_cache' is true.
// Because each FE fetch the meta data independently.
db.ifPresent(e -> {
e.setUnInitialized(invalidCache);
LOG.info("refresh database {} in catalog {} with invalidCache {}", e.getFullName(),
catalog.getName(), invalidCache);
});
if (!db.isPresent()) {
LOG.warn("failed to find db when replaying refresh db: {}", log.debugForRefreshDb());
} else {
refreshDbInternal(db.get(), log.isInvalidCache());
}
}

// Refresh table
public void handleRefreshTable(RefreshTableStmt stmt) throws UserException {
String catalogName = stmt.getCtl();
String dbName = stmt.getDbName();
String tableName = stmt.getTblName();
refreshTable(catalogName, dbName, tableName, false);
private void refreshDbInternal(ExternalDatabase db, boolean invalidCache) {
db.setUnInitialized();
if (invalidCache) {
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(db.getCatalog().getId(), db.getFullName());
}
LOG.info("refresh database {} in catalog {} with invalidCache {}", db.getFullName(),
db.getCatalog().getName(), invalidCache);
}

public void refreshTable(String catalogName, String dbName, String tableName, boolean ignoreIfNotExists)
// Refresh table
public void handleRefreshTable(String catalogName, String dbName, String tableName, boolean ignoreIfNotExists)
throws DdlException {
Env env = Env.getCurrentEnv();
CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
Expand All @@ -153,33 +151,41 @@ public void refreshTable(String catalogName, String dbName, String tableName, bo
}
return;
}
refreshTableInternal(catalog, db, table, 0);
refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, 0);

ExternalObjectLog log = new ExternalObjectLog();
log.setCatalogId(catalog.getId());
log.setDbId(db.getId());
log.setTableId(table.getId());
ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(catalog.getId(), db.getFullName(),
table.getName());
Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log);
}

public void replayRefreshTable(ExternalObjectLog log) {
ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(log.getCatalogId());
if (catalog == null) {
LOG.warn("failed to find catalog replaying refresh table {}", log.getCatalogId());
LOG.warn("failed to find catalog when replaying refresh table: {}", log.debugForRefreshTable());
return;
}
Optional<ExternalDatabase<? extends ExternalTable>> db = catalog.getDbForReplay(log.getDbId());
Optional<ExternalDatabase<? extends ExternalTable>> db;
if (!Strings.isNullOrEmpty(log.getDbName())) {
db = catalog.getDbForReplay(log.getDbName());
} else {
db = catalog.getDbForReplay(log.getDbId());
}
// See comment in refreshDbInternal for why db and table may be null.
if (!db.isPresent()) {
LOG.warn("failed to find db replaying refresh table {}", log.getDbId());
LOG.warn("failed to find db when replaying refresh table: {}", log.debugForRefreshTable());
return;
}
Optional<? extends ExternalTable> table = db.get().getTableForReplay(log.getTableId());
Optional<? extends ExternalTable> table;
if (!Strings.isNullOrEmpty(log.getTableName())) {
table = db.get().getTableForReplay(log.getTableName());
} else {
table = db.get().getTableForReplay(log.getTableId());
}
if (!table.isPresent()) {
LOG.warn("failed to find table replaying refresh table {}", log.getTableId());
LOG.warn("failed to find table when replaying refresh table: {}", log.debugForRefreshTable());
return;
}
refreshTableInternal(catalog, db.get(), table.get(), log.getLastUpdateTime());
refreshTableInternal(db.get(), table.get(), log.getLastUpdateTime());
}

public void refreshExternalTableFromEvent(String catalogName, String dbName, String tableName,
Expand All @@ -200,19 +206,17 @@ public void refreshExternalTableFromEvent(String catalogName, String dbName, Str
if (table == null) {
return;
}
refreshTableInternal(catalog, db, table, updateTime);
refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime);
}

private void refreshTableInternal(CatalogIf catalog, DatabaseIf db, TableIf table, long updateTime) {
if (table instanceof ExternalTable) {
((ExternalTable) table).unsetObjectCreated();
}
Env.getCurrentEnv().getExtMetaCacheMgr()
.invalidateTableCache(catalog.getId(), db.getFullName(), table.getName());
public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime) {
table.unsetObjectCreated();
if (table instanceof HMSExternalTable && updateTime > 0) {
((HMSExternalTable) table).setEventUpdateTime(updateTime);
}
LOG.info("refresh table {} from db {} in catalog {}", table.getName(), db.getFullName(), catalog.getName());
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table);
LOG.info("refresh table {}, id {} from db {} in catalog {}",
table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName());
}

// Refresh partition
Expand Down Expand Up @@ -245,8 +249,7 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam
return;
}

Env.getCurrentEnv().getExtMetaCacheMgr().invalidatePartitionsCache(
catalog.getId(), db.getFullName(), table.getName(), partitionNames);
Env.getCurrentEnv().getExtMetaCacheMgr().invalidatePartitionsCache((ExternalTable) table, partitionNames);
((HMSExternalTable) table).setEventUpdateTime(updateTime);
}

Expand Down
Loading