Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,6 @@
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import lombok.Data;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
Expand Down Expand Up @@ -107,7 +106,6 @@
/**
* The abstract class for all types of external catalogs.
*/
@Data
public abstract class ExternalCatalog
implements CatalogIf<ExternalDatabase<? extends ExternalTable>>, Writable, GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(ExternalCatalog.class);
Expand Down Expand Up @@ -364,7 +362,7 @@ public boolean isInitialized() {
// check if all required properties are set when creating catalog
public void checkProperties() throws DdlException {
// check refresh parameter of catalog
Map<String, String> properties = getCatalogProperty().getProperties();
Map<String, String> properties = catalogProperty.getProperties();
if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) {
try {
int metadataRefreshIntervalSec = Integer.parseInt(
Expand Down Expand Up @@ -397,7 +395,7 @@ public void checkProperties() throws DdlException {
* isDryRun: if true, it will try to create the custom access controller, but will not add it to the access manager.
*/
public void initAccessController(boolean isDryRun) {
Map<String, String> properties = getCatalogProperty().getProperties();
Map<String, String> properties = catalogProperty.getProperties();
// 1. get access controller class name
String className = properties.getOrDefault(CatalogMgr.ACCESS_CONTROLLER_CLASS_PROP, "");
if (Strings.isNullOrEmpty(className)) {
Expand Down Expand Up @@ -557,7 +555,7 @@ private List<Pair<String, String>> getFilteredDatabaseNames() {
* @param invalidCache if {@code true}, the catalog cache will be invalidated
* and reloaded during the refresh process.
*/
public void resetToUninitialized(boolean invalidCache) {
public synchronized void resetToUninitialized(boolean invalidCache) {
this.objectCreated = false;
this.initialized = false;
synchronized (this.propLock) {
Expand Down Expand Up @@ -1009,6 +1007,14 @@ public void addDatabaseForTest(ExternalDatabase<? extends ExternalTable> db) {
dbNameToId.put(ClusterNamespace.getNameFromFullName(db.getFullName()), db.getId());
}

/**
* Set the initialized status for testing purposes only.
* This method should only be used in test cases.
*/
public void setInitializedForTest(boolean initialized) {
this.initialized = initialized;
}

@Override
public void createDb(CreateDbStmt stmt) throws DdlException {
makeSureInitialized();
Expand Down Expand Up @@ -1312,4 +1318,24 @@ public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
public ThreadPoolExecutor getThreadPoolExecutor() {
return threadPoolWithPreAuth;
}

public CatalogProperty getCatalogProperty() {
return catalogProperty;
}

public Optional<Boolean> getUseMetaCache() {
return useMetaCache;
}

public Map<Pair<String, String>, String> getTableAutoAnalyzePolicy() {
return tableAutoAnalyzePolicy;
}

public TransactionManager getTransactionManager() {
return transactionManager;
}

public ThreadPoolExecutor getThreadPoolWithPreAuth() {
return threadPoolWithPreAuth;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ public void setTableExtCatalog(ExternalCatalog extCatalog) {
}
}

public void setUnInitialized(boolean invalidCache) {
public synchronized void setUnInitialized(boolean invalidCache) {
this.initialized = false;
this.invalidCacheInInit = invalidCache;
if (extCatalog.getUseMetaCache().isPresent()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import com.google.common.base.Objects;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.logging.log4j.LogManager;
Expand All @@ -64,7 +63,6 @@
* External table represent tables that are not self-managed by Doris.
* Such as tables from hive, iceberg, es, etc.
*/
@Getter
public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(ExternalTable.class);

Expand Down Expand Up @@ -466,4 +464,36 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hashCode(name, db);
}

public long getSchemaUpdateTime() {
return schemaUpdateTime;
}

public long getDbId() {
return dbId;
}

public boolean isObjectCreated() {
return objectCreated;
}

public ExternalCatalog getCatalog() {
return catalog;
}

public ExternalDatabase getDb() {
return db;
}

public long getTimestamp() {
return timestamp;
}

public String getDbName() {
return dbName;
}

public TableAttributes getTableAttributes() {
return tableAttributes;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ protected void initLocalObjectsImpl() {
}

@Override
public void resetToUninitialized(boolean invalidCache) {
public synchronized void resetToUninitialized(boolean invalidCache) {
super.resetToUninitialized(invalidCache);
if (metadataOps != null) {
metadataOps.close();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.protobuf.ByteString;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -64,7 +63,6 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

@Getter
public class JdbcExternalCatalog extends ExternalCatalog {
private static final Logger LOG = LogManager.getLogger(JdbcExternalCatalog.class);

Expand Down Expand Up @@ -130,7 +128,7 @@ public void setDefaultPropsIfMissing(boolean isReplay) {
}

@Override
public void resetToUninitialized(boolean invalidCache) {
public synchronized void resetToUninitialized(boolean invalidCache) {
super.resetToUninitialized(invalidCache);
this.identifierMapping = new JdbcIdentifierMapping(
(Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()),
Expand Down Expand Up @@ -448,4 +446,8 @@ private JdbcTable getTestConnectionJdbcTable(JdbcClient testClient) throws DdlEx
public ExternalFunctionRules getFunctionRules() {
return functionRules;
}

public IdentifierMapping getIdentifierMapping() {
return identifierMapping;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public static void beforeClass() throws Throwable {
if (icebergCatalog.getUseMetaCache().get()) {
icebergCatalog.makeSureInitialized();
} else {
icebergCatalog.setInitialized(true);
icebergCatalog.setInitializedForTest(true);
}

// create db
Expand All @@ -82,7 +82,7 @@ public static void beforeClass() throws Throwable {
if (icebergCatalog.getUseMetaCache().get()) {
icebergCatalog.makeSureInitialized();
} else {
icebergCatalog.setInitialized(true);
icebergCatalog.setInitializedForTest(true);
}
IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName);
icebergCatalog.addDatabaseForTest(db);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public void testGetPaimonTable() {
HashMap<String, String> props = new HashMap<>();
props.put("warehouse", "not_exist");
PaimonExternalCatalog catalog = new PaimonFileExternalCatalog(1, "name", "resource", props, "comment");
catalog.setInitialized(true);
catalog.setInitializedForTest(true);

try {
catalog.getPaimonTable("dbName", "tblName");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public HMSCachedClient getClient() {
mockDifferLocationTable(location);

HMSExternalCatalog hmsExternalCatalog = new HMSExternalCatalog();
hmsExternalCatalog.setInitialized(true);
hmsExternalCatalog.setInitializedForTest(true);
HMSExternalDatabase db = new HMSExternalDatabase(hmsExternalCatalog, 10000, "hive_db1", "hive_db1");
HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsExternalCatalog, db);
HiveTableSink hiveTableSink = new HiveTableSink(tbl);
Expand Down
Loading