diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 6e3963139a8366..3cc4ab759fbfeb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -79,7 +79,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; -import lombok.Data; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -107,7 +106,6 @@ /** * The abstract class for all types of external catalogs. */ -@Data public abstract class ExternalCatalog implements CatalogIf>, Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(ExternalCatalog.class); @@ -364,7 +362,7 @@ public boolean isInitialized() { // check if all required properties are set when creating catalog public void checkProperties() throws DdlException { // check refresh parameter of catalog - Map properties = getCatalogProperty().getProperties(); + Map properties = catalogProperty.getProperties(); if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) { try { int metadataRefreshIntervalSec = Integer.parseInt( @@ -397,7 +395,7 @@ public void checkProperties() throws DdlException { * isDryRun: if true, it will try to create the custom access controller, but will not add it to the access manager. */ public void initAccessController(boolean isDryRun) { - Map properties = getCatalogProperty().getProperties(); + Map properties = catalogProperty.getProperties(); // 1. get access controller class name String className = properties.getOrDefault(CatalogMgr.ACCESS_CONTROLLER_CLASS_PROP, ""); if (Strings.isNullOrEmpty(className)) { @@ -557,7 +555,7 @@ private List> getFilteredDatabaseNames() { * @param invalidCache if {@code true}, the catalog cache will be invalidated * and reloaded during the refresh process. */ - public void resetToUninitialized(boolean invalidCache) { + public synchronized void resetToUninitialized(boolean invalidCache) { this.objectCreated = false; this.initialized = false; synchronized (this.propLock) { @@ -1009,6 +1007,14 @@ public void addDatabaseForTest(ExternalDatabase db) { dbNameToId.put(ClusterNamespace.getNameFromFullName(db.getFullName()), db.getId()); } + /** + * Set the initialized status for testing purposes only. + * This method should only be used in test cases. + */ + public void setInitializedForTest(boolean initialized) { + this.initialized = initialized; + } + @Override public void createDb(CreateDbStmt stmt) throws DdlException { makeSureInitialized(); @@ -1312,4 +1318,24 @@ public void notifyPropertiesUpdated(Map updatedProps) { public ThreadPoolExecutor getThreadPoolExecutor() { return threadPoolWithPreAuth; } + + public CatalogProperty getCatalogProperty() { + return catalogProperty; + } + + public Optional getUseMetaCache() { + return useMetaCache; + } + + public Map, String> getTableAutoAnalyzePolicy() { + return tableAutoAnalyzePolicy; + } + + public TransactionManager getTransactionManager() { + return transactionManager; + } + + public ThreadPoolExecutor getThreadPoolWithPreAuth() { + return threadPoolWithPreAuth; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 307da348069ec6..5b0370ec290204 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -134,7 +134,7 @@ public void setTableExtCatalog(ExternalCatalog extCatalog) { } } - public void setUnInitialized(boolean invalidCache) { + public synchronized void setUnInitialized(boolean invalidCache) { this.initialized = false; this.invalidCacheInInit = invalidCache; if (extCatalog.getUseMetaCache().isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 60a1f172978124..8e999df2aef03b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -45,7 +45,6 @@ import com.google.common.base.Objects; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; -import lombok.Getter; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.logging.log4j.LogManager; @@ -64,7 +63,6 @@ * External table represent tables that are not self-managed by Doris. * Such as tables from hive, iceberg, es, etc. */ -@Getter public class ExternalTable implements TableIf, Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(ExternalTable.class); @@ -466,4 +464,36 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hashCode(name, db); } + + public long getSchemaUpdateTime() { + return schemaUpdateTime; + } + + public long getDbId() { + return dbId; + } + + public boolean isObjectCreated() { + return objectCreated; + } + + public ExternalCatalog getCatalog() { + return catalog; + } + + public ExternalDatabase getDb() { + return db; + } + + public long getTimestamp() { + return timestamp; + } + + public String getDbName() { + return dbName; + } + + public TableAttributes getTableAttributes() { + return tableAttributes; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 88a310d5f1dde3..ee7a3d50f6b47e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -196,7 +196,7 @@ protected void initLocalObjectsImpl() { } @Override - public void resetToUninitialized(boolean invalidCache) { + public synchronized void resetToUninitialized(boolean invalidCache) { super.resetToUninitialized(invalidCache); if (metadataOps != null) { metadataOps.close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 4899cb2ec1cef1..85e3b29e1a3e55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -51,7 +51,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.protobuf.ByteString; -import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -64,7 +63,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -@Getter public class JdbcExternalCatalog extends ExternalCatalog { private static final Logger LOG = LogManager.getLogger(JdbcExternalCatalog.class); @@ -130,7 +128,7 @@ public void setDefaultPropsIfMissing(boolean isReplay) { } @Override - public void resetToUninitialized(boolean invalidCache) { + public synchronized void resetToUninitialized(boolean invalidCache) { super.resetToUninitialized(invalidCache); this.identifierMapping = new JdbcIdentifierMapping( (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), @@ -448,4 +446,8 @@ private JdbcTable getTestConnectionJdbcTable(JdbcClient testClient) throws DdlEx public ExternalFunctionRules getFunctionRules() { return functionRules; } + + public IdentifierMapping getIdentifierMapping() { + return identifierMapping; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 2300ece6253b5f..45fe77387835cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -72,7 +72,7 @@ public static void beforeClass() throws Throwable { if (icebergCatalog.getUseMetaCache().get()) { icebergCatalog.makeSureInitialized(); } else { - icebergCatalog.setInitialized(true); + icebergCatalog.setInitializedForTest(true); } // create db @@ -82,7 +82,7 @@ public static void beforeClass() throws Throwable { if (icebergCatalog.getUseMetaCache().get()) { icebergCatalog.makeSureInitialized(); } else { - icebergCatalog.setInitialized(true); + icebergCatalog.setInitializedForTest(true); } IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName); icebergCatalog.addDatabaseForTest(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java index 35dd64515b6ec9..da66b3c5ba9a24 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java @@ -31,7 +31,7 @@ public void testGetPaimonTable() { HashMap props = new HashMap<>(); props.put("warehouse", "not_exist"); PaimonExternalCatalog catalog = new PaimonFileExternalCatalog(1, "name", "resource", props, "comment"); - catalog.setInitialized(true); + catalog.setInitializedForTest(true); try { catalog.getPaimonTable("dbName", "tblName"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java index b57bbcb51a2772..602b2c8a18ade8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java @@ -88,7 +88,7 @@ public HMSCachedClient getClient() { mockDifferLocationTable(location); HMSExternalCatalog hmsExternalCatalog = new HMSExternalCatalog(); - hmsExternalCatalog.setInitialized(true); + hmsExternalCatalog.setInitializedForTest(true); HMSExternalDatabase db = new HMSExternalDatabase(hmsExternalCatalog, 10000, "hive_db1", "hive_db1"); HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsExternalCatalog, db); HiveTableSink hiveTableSink = new HiveTableSink(tbl);