From 2c63cc39e62b54450f2326707bdbfa44f9dd537c Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 10:41:52 +0800 Subject: [PATCH 01/20] save --- .../apache/doris/common/NereidsSqlCacheManager.java | 13 ------------- .../org/apache/doris/nereids/StatementContext.java | 1 - 2 files changed, 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index f3643046cade04..903942912044e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -328,19 +328,6 @@ private boolean nondeterministicFunctionChanged( return false; } - private boolean isValidDbAndTable(TableIf tableIf, Env env) { - return getTableFromEnv(tableIf, env) != null; - } - - private TableIf getTableFromEnv(TableIf tableIf, Env env) { - Optional db = env.getInternalCatalog().getDb(tableIf.getDatabase().getId()); - if (!db.isPresent()) { - return null; - } - Optional table = db.get().getTable(tableIf.getId()); - return table.orElse(null); - } - private Optional invalidateCache(String key) { sqlCache.invalidate(key); return Optional.empty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 5baa59a35dec1b..423d7fd8902fa2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -112,7 +112,6 @@ public class StatementContext implements Closeable { private final Map rewrittenCteConsumer = new HashMap<>(); private final Set viewDdlSqlSet = Sets.newHashSet(); private final SqlCacheContext sqlCacheContext; - private Map> checkedPrivilegedTableAndUsedColumns = Maps.newLinkedHashMap(); // collect all hash join conditions to compute node connectivity in join graph private final List joinFilters = new ArrayList<>(); From 2ebb2b855456a2deb5cacaca0923fbe5ff8a3bdc Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 10:51:36 +0800 Subject: [PATCH 02/20] save --- .../org/apache/doris/common/NereidsSqlCacheManager.java | 9 --------- gensrc/proto/internal_service.proto | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index 903942912044e5..c77df5b21718f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -18,12 +18,10 @@ package org.apache.doris.common; import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; import org.apache.doris.datasource.CatalogIf; @@ -113,13 +111,6 @@ public void tryAddCache( } } - /** invalidateCache */ - public void invalidateCache(ConnectContext connectContext, String sql) { - UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); - String key = currentUserIdentity.toString() + ":" + sql.trim(); - sqlCache.invalidate(key); - } - /** tryParseSql */ public Optional tryParseSql(ConnectContext connectContext, String sql) { UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 3d97c6228433d7..a519dfe561fd49 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -342,6 +342,8 @@ message PCacheParam { optional int64 last_version = 2; optional int64 last_version_time = 3; optional int64 partition_num = 4; + optional string cache_owner = 5; + optional int64 cache_version = 6; }; message PCacheValue { From 4b8d6ac949989a7be719dc2ee0fab3b72c481280 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 12:30:45 +0800 Subject: [PATCH 03/20] save --- .../java/org/apache/doris/common/Config.java | 16 +++++-- .../org/apache/doris/common/ConfigBase.java | 12 +++++ .../doris/common/NereidsSqlCacheManager.java | 47 ++++++++++++++++--- .../org/apache/doris/qe/cache/CacheProxy.java | 1 - 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index d318b77aa416dd..c6fe69756e5b84 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1300,7 +1300,11 @@ public class Config extends ConfigBase { * Minimum interval between last version when caching results, * This parameter distinguishes between offline and real-time updates */ - @ConfField(mutable = true, masterOnly = false) + @ConfField( + mutable = true, + masterOnly = false, + callbackClassString = "org.apache.doris.common.NereidsSqlCacheManager$UpdateConfig" + ) public static int cache_last_version_interval_second = 30; /** @@ -2011,10 +2015,16 @@ public class Config extends ConfigBase { /** * the plan cache num which can be reused for the next query */ - @ConfField(mutable = false, varType = VariableAnnotation.EXPERIMENTAL, description = { + @ConfField( + mutable = true, + varType = VariableAnnotation.EXPERIMENTAL, + callbackClassString = "org.apache.doris.common.NereidsSqlCacheManager$UpdateConfig", + description = { "当前默认设置为 100，用来控制控制NereidsSqlCacheManager管理的sql cache数量。", "Now default set to 100, this config is used to control the number of " - + "sql cache managed by NereidsSqlCacheManager"}) + + "sql cache managed by NereidsSqlCacheManager" + } + ) public static int sql_cache_manage_num = 100; /** diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java index 2411ff997c7d0b..dd66fdcb0f593b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java @@ -56,6 +56,8 @@ public class ConfigBase { Class callback() default DefaultConfHandler.class; + String callbackClassString() default ""; + // description for this config item. // There should be 2 elements in the array. // The first element is the description in Chinese. @@ -329,6 +331,16 @@ public static synchronized void setMutableConfig(String key, String value) throw throw new ConfigException("Failed to set config '" + key + "'. err: " + e.getMessage()); } + String callbackClassString = anno.callbackClassString(); + if (!Strings.isNullOrEmpty(callbackClassString)) { + try { + ConfHandler confHandler = (ConfHandler) Class.forName(anno.callbackClassString()).newInstance(); + confHandler.handle(field, value); + } catch (Exception e) { + throw new ConfigException("Failed to set config '" + key + "'. err: " + e.getMessage()); + } + } + LOG.info("set config {} to {}", key, value); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index c77df5b21718f1..ff8b3754f24aaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; +import org.apache.doris.common.ConfigBase.DefaultConfHandler; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.privilege.DataMaskPolicy; @@ -57,6 +58,7 @@ import com.google.common.collect.ImmutableList; import org.apache.commons.collections.CollectionUtils; +import java.lang.reflect.Field; import java.time.Duration; import java.util.List; import java.util.Map.Entry; @@ -68,10 +70,10 @@ public class NereidsSqlCacheManager { // key: : // value: CacheAnalyzer - private final Cache sqlCache; + private volatile Cache sqlCaches; public NereidsSqlCacheManager(int sqlCacheNum, long cacheIntervalSeconds) { - sqlCache = Caffeine.newBuilder() + sqlCaches = Caffeine.newBuilder() .maximumSize(sqlCacheNum) .expireAfterAccess(Duration.ofSeconds(cacheIntervalSeconds)) // auto evict cache when jvm memory too low @@ -79,6 +81,29 @@ public NereidsSqlCacheManager(int sqlCacheNum, long cacheIntervalSeconds) { .build(); } + public static synchronized void updateConfig() { + Env currentEnv = Env.getCurrentEnv(); + if (currentEnv == null) { + return; + } + NereidsSqlCacheManager sqlCacheManager = currentEnv.getSqlCacheManager(); + if (sqlCacheManager == null) { + return; + } + + int sqlCacheManageNum = Config.sql_cache_manage_num; + int cacheIntervalSecond = Config.cache_last_version_interval_second; + + Cache sqlCaches = Caffeine.newBuilder() + .maximumSize(sqlCacheManageNum) + .expireAfterAccess(Duration.ofSeconds(cacheIntervalSecond)) + // auto evict cache when jvm memory too low + .softValues() + .build(); + sqlCaches.putAll(sqlCacheManager.sqlCaches.asMap()); + sqlCacheManager.sqlCaches = sqlCaches; + } + /** tryAddCache */ public void tryAddCache( ConnectContext connectContext, String sql, @@ -94,7 +119,7 @@ public void tryAddCache( UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); String key = currentUserIdentity.toString() + ":" + sql.trim(); if (analyzer.getCache() instanceof SqlCache - && (currentMissParseSqlFromSqlCache || sqlCache.getIfPresent(key) == null)) { + && (currentMissParseSqlFromSqlCache || sqlCaches.getIfPresent(key) == null)) { SqlCache cache = (SqlCache) analyzer.getCache(); sqlCacheContext.setCacheKeyMd5(cache.getOrComputeCacheMd5()); sqlCacheContext.setSumOfPartitionNum(cache.getSumOfPartitionNum()); @@ -107,7 +132,7 @@ public void tryAddCache( sqlCacheContext.addScanTable(scanTable); } - sqlCache.put(key, sqlCacheContext); + sqlCaches.put(key, sqlCacheContext); } } @@ -116,7 +141,7 @@ public Optional tryParseSql(ConnectContext connectContext, Stri UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); Env env = connectContext.getEnv(); String key = currentUserIdentity.toString() + ":" + sql.trim(); - SqlCacheContext sqlCacheContext = sqlCache.getIfPresent(key); + SqlCacheContext sqlCacheContext = sqlCaches.getIfPresent(key); if (sqlCacheContext == null) { return Optional.empty(); } @@ -320,7 +345,7 @@ private boolean nondeterministicFunctionChanged( } private Optional invalidateCache(String key) { - sqlCache.invalidate(key); + sqlCaches.invalidate(key); return Optional.empty(); } @@ -335,4 +360,14 @@ private TableIf findTableIf(Env env, FullTableName fullTableName) { } return db.get().getTable(fullTableName.table).orElse(null); } + + // used in Config.sql_cache_manage_num.callbackClassString and + // Config.cache_last_version_interval_second.callbackClassString, don't remove it + public static class UpdateConfig extends DefaultConfHandler { + @Override + public void handle(Field field, String confVal) throws Exception { + super.handle(field, confVal); + NereidsSqlCacheManager.updateConfig(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java index 20d6f5d0d02401..dd0e0ce6bcdb77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java @@ -60,7 +60,6 @@ public abstract InternalService.PFetchCacheResult fetchCache(InternalService.PFe public abstract void clearCache(InternalService.PClearCacheRequest clearRequest); - public static Types.PUniqueId getMd5(String str) { MessageDigest msgDigest; final byte[] digest; From 36e7d99796f974af1a543b00fff68739e62debd4 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 15:58:50 +0800 Subject: [PATCH 04/20] save --- .../doris/common/NereidsSqlCacheManager.java | 6 ++-- .../org/apache/doris/qe/StmtExecutor.java | 32 ++++++------------- .../apache/doris/qe/cache/CacheAnalyzer.java | 11 +++++-- .../doris/qe/cache/RowBatchBuilder.java | 14 ++++++-- .../org/apache/doris/qe/cache/SqlCache.java | 6 ++-- 5 files changed, 36 insertions(+), 33 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index ff8b3754f24aaa..a98fe0ab9a4d70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -52,6 +52,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.cache.CacheAnalyzer; import org.apache.doris.qe.cache.SqlCache; +import org.apache.doris.service.FrontendOptions; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -176,6 +177,7 @@ public Optional tryParseSql(ConnectContext connectContext, Stri } try { + String localHostAddress = FrontendOptions.getLocalHostAddress(); Status status = new Status(); InternalService.PFetchCacheResult cacheData = SqlCache.getCacheData(sqlCacheContext.getCacheProxy(), @@ -195,9 +197,9 @@ public Optional tryParseSql(ConnectContext connectContext, Stri sqlCacheContext.getResultExprs(), cacheValues, backendAddress, cachedPlan); return Optional.of(logicalSqlCache); } - return Optional.empty(); + return invalidateCache(key); } catch (Throwable t) { - return Optional.empty(); + return invalidateCache(key); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 19faa555999139..7120774b6fd7b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -106,7 +106,6 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.NereidsException; -import org.apache.doris.common.NereidsSqlCacheManager; import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.profile.Profile; @@ -1667,32 +1666,19 @@ private boolean sendCachedValues(MysqlChannel channel, List buildCacheTableList() { } public InternalService.PFetchCacheResult getCacheData() throws UserException { - if (parsedStmt instanceof LogicalPlanAdapter) { + boolean isNereids = parsedStmt instanceof LogicalPlanAdapter; + if (isNereids) { cacheMode = innerCheckCacheModeForNereids(0); } else if (parsedStmt instanceof SelectStmt) { cacheMode = innerCheckCacheMode(0); @@ -526,8 +527,12 @@ public InternalService.PFetchCacheResult getCacheData() throws UserException { return null; } Status status = new Status(); - InternalService.PFetchCacheResult cacheResult = cache.getCacheData(status); - if (status.ok() && cacheResult != null && cacheResult.getStatus() == InternalService.PCacheStatus.CACHE_OK) { + InternalService.PFetchCacheResult cacheResult = null; + if (!isNereids) { + cacheResult = cache.getCacheData(status); + } + if (!isNereids && status.ok() + && cacheResult != null && cacheResult.getStatus() == InternalService.PCacheStatus.CACHE_OK) { int rowCount = 0; int dataSize = 0; for (InternalService.PCacheValue value : cacheResult.getValuesList()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java index 5cab6f74bb3411..bfff2a744819ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.proto.InternalService; import org.apache.doris.qe.RowBatch; +import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TResultBatch; import com.google.common.collect.Lists; @@ -130,9 +131,16 @@ public InternalService.PUpdateCacheRequest buildSqlUpdateRequest( .setLastVersion(lastVersion) .setLastVersionTime(lastestTime) .setPartitionNum(partitionNum) - .build()).setDataSize(dataSize).addAllRows( - rowList.stream().map(row -> ByteString.copyFrom(row)) - .collect(Collectors.toList()))).build(); + .setCacheOwner(FrontendOptions.getLocalHostAddress()) + .build() + ) + .setDataSize(dataSize) + .addAllRows( + rowList.stream() + .map(row -> ByteString.copyFrom(row)) + .collect(Collectors.toList()) + ) + ).build(); return updateRequest; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index a7351a306d2e2d..0e51ece43b592a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -142,8 +142,10 @@ public void updateCache() { } InternalService.PUpdateCacheRequest updateRequest = - rowBatchBuilder.buildSqlUpdateRequest(getSqlWithViewStmt(), latestTable.latestPartitionId, - latestTable.latestPartitionVersion, latestTable.latestPartitionTime, + rowBatchBuilder.buildSqlUpdateRequest(getSqlWithViewStmt(), + latestTable.latestPartitionId, + latestTable.latestPartitionVersion, + latestTable.latestPartitionTime, latestTable.sumOfPartitionNum ); if (updateRequest.getValuesCount() > 0) { From 009b80c9d215a0a6f5bc5c410c2067b8aa18ca7b Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 16:01:57 +0800 Subject: [PATCH 05/20] save --- .../src/main/java/org/apache/doris/qe/StmtExecutor.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 7120774b6fd7b8..44770b5bb3ee96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -1666,9 +1666,7 @@ private boolean sendCachedValues(MysqlChannel channel, List Date: Mon, 15 Apr 2024 18:39:32 +0800 Subject: [PATCH 06/20] fix clear --- .../java/org/apache/doris/common/Config.java | 6 ++--- .../doris/common/NereidsSqlCacheManager.java | 26 ++++++++++--------- .../apache/doris/qe/cache/CacheBeProxy.java | 1 - .../org/apache/doris/qe/cache/SqlCache.java | 11 ++++++++ 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c6fe69756e5b84..efe0285f5ee9b7 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2020,9 +2020,9 @@ public class Config extends ConfigBase { varType = VariableAnnotation.EXPERIMENTAL, callbackClassString = "org.apache.doris.common.NereidsSqlCacheManager$UpdateConfig", description = { - "当前默认设置为 100，用来控制控制NereidsSqlCacheManager管理的sql cache数量。", - "Now default set to 100, this config is used to control the number of " - + "sql cache managed by NereidsSqlCacheManager" + "当前默认设置为 100，用来控制控制NereidsSqlCacheManager管理的sql cache数量。", + "Now default set to 100, this config is used to control the number of " + + "sql cache managed by NereidsSqlCacheManager" } ) public static int sql_cache_manage_num = 100; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index a98fe0ab9a4d70..efff57c064832b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -49,10 +49,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; import org.apache.doris.proto.InternalService; +import org.apache.doris.proto.InternalService.PClearType; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.cache.CacheAnalyzer; import org.apache.doris.qe.cache.SqlCache; -import org.apache.doris.service.FrontendOptions; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -151,33 +151,32 @@ public Optional tryParseSql(ConnectContext connectContext, Stri // check table and view and their columns authority if (privilegeChanged(connectContext, env, sqlCacheContext)) { - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } if (tablesOrDataChanged(env, sqlCacheContext)) { - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } if (viewsChanged(env, sqlCacheContext)) { - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } if (usedVariablesChanged(sqlCacheContext)) { - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } LogicalEmptyRelation whateverPlan = new LogicalEmptyRelation(new RelationId(0), ImmutableList.of()); if (nondeterministicFunctionChanged(whateverPlan, connectContext, sqlCacheContext)) { - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } // table structure and data not changed, now check policy if (rowPoliciesChanged(currentUserIdentity, env, sqlCacheContext)) { - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } if (dataMaskPoliciesChanged(currentUserIdentity, env, sqlCacheContext)) { - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } try { - String localHostAddress = FrontendOptions.getLocalHostAddress(); Status status = new Status(); InternalService.PFetchCacheResult cacheData = SqlCache.getCacheData(sqlCacheContext.getCacheProxy(), @@ -197,9 +196,9 @@ public Optional tryParseSql(ConnectContext connectContext, Stri sqlCacheContext.getResultExprs(), cacheValues, backendAddress, cachedPlan); return Optional.of(logicalSqlCache); } - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } catch (Throwable t) { - return invalidateCache(key); + return invalidateCache(key, sqlCacheContext); } } @@ -346,8 +345,11 @@ private boolean nondeterministicFunctionChanged( return false; } - private Optional invalidateCache(String key) { + private Optional invalidateCache(String key, SqlCacheContext sqlCacheContext) { sqlCaches.invalidate(key); + SqlCache.clearCache( + sqlCacheContext.getCacheProxy(), sqlCacheContext.getCacheKeyMd5(), PClearType.CLEAR_SQL_KEY + ); return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java index 9be5aacef110d8..70ec56e79dbd55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java @@ -124,7 +124,6 @@ protected boolean clearCache(InternalService.PClearCacheRequest request, Backend backend, int timeoutMs, Status status) { TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort()); try { - request = request.toBuilder().setClearType(InternalService.PClearType.CLEAR_ALL).build(); LOG.info("clear all backend cache, backendId {}", backend.getId()); Future future = BackendServiceProxy.getInstance().clearCache(address, request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index 0e51ece43b592a..247179a062b759 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -22,6 +22,8 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.metric.MetricRepo; import org.apache.doris.proto.InternalService; +import org.apache.doris.proto.InternalService.PClearCacheRequest; +import org.apache.doris.proto.InternalService.PClearType; import org.apache.doris.proto.Types.PUniqueId; import org.apache.doris.qe.RowBatch; import org.apache.doris.system.Backend; @@ -92,6 +94,15 @@ public static Backend findCacheBe(PUniqueId cacheMd5) { return CacheCoordinator.getInstance().findBackend(cacheMd5); } + public static void clearCache(CacheProxy proxy, PUniqueId cacheKeyMd5, PClearType clearType) { + PClearCacheRequest request = PClearCacheRequest.newBuilder() + .setSqlKey(cacheKeyMd5) + .setClearType(clearType) + .build(); + + proxy.clearCache(request); + } + public static InternalService.PFetchCacheResult getCacheData(CacheProxy proxy, PUniqueId cacheKeyMd5, long latestPartitionId, long latestPartitionVersion, long latestPartitionTime, long sumOfPartitionNum, Status status) { From 7aa517e70235bb075d27e3e21c950f1c75add016 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 19:28:12 +0800 Subject: [PATCH 07/20] save --- .../doris/common/NereidsSqlCacheManager.java | 25 ++++++++----------- gensrc/proto/internal_service.proto | 2 -- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index efff57c064832b..74c60c8648db30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -70,7 +70,7 @@ /** NereidsSqlCacheManager */ public class NereidsSqlCacheManager { // key: : - // value: CacheAnalyzer + // value: SqlCacheContext private volatile Cache sqlCaches; public NereidsSqlCacheManager(int sqlCacheNum, long cacheIntervalSeconds) { @@ -151,29 +151,29 @@ public Optional tryParseSql(ConnectContext connectContext, Stri // check table and view and their columns authority if (privilegeChanged(connectContext, env, sqlCacheContext)) { - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } if (tablesOrDataChanged(env, sqlCacheContext)) { - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } if (viewsChanged(env, sqlCacheContext)) { - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } if (usedVariablesChanged(sqlCacheContext)) { - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } LogicalEmptyRelation whateverPlan = new LogicalEmptyRelation(new RelationId(0), ImmutableList.of()); if (nondeterministicFunctionChanged(whateverPlan, connectContext, sqlCacheContext)) { - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } // table structure and data not changed, now check policy if (rowPoliciesChanged(currentUserIdentity, env, sqlCacheContext)) { - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } if (dataMaskPoliciesChanged(currentUserIdentity, env, sqlCacheContext)) { - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } try { @@ -196,9 +196,9 @@ public Optional tryParseSql(ConnectContext connectContext, Stri sqlCacheContext.getResultExprs(), cacheValues, backendAddress, cachedPlan); return Optional.of(logicalSqlCache); } - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } catch (Throwable t) { - return invalidateCache(key, sqlCacheContext); + return invalidateCache(key); } } @@ -345,11 +345,8 @@ private boolean nondeterministicFunctionChanged( return false; } - private Optional invalidateCache(String key, SqlCacheContext sqlCacheContext) { + private Optional invalidateCache(String key) { sqlCaches.invalidate(key); - SqlCache.clearCache( - sqlCacheContext.getCacheProxy(), sqlCacheContext.getCacheKeyMd5(), PClearType.CLEAR_SQL_KEY - ); return Optional.empty(); } diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index a519dfe561fd49..3d97c6228433d7 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -342,8 +342,6 @@ message PCacheParam { optional int64 last_version = 2; optional int64 last_version_time = 3; optional int64 partition_num = 4; - optional string cache_owner = 5; - optional int64 cache_version = 6; }; message PCacheValue { From 0f2833566197bc1bf6b02b9e1b4046ca777620ca Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 19:54:09 +0800 Subject: [PATCH 08/20] save --- .../doris/common/NereidsSqlCacheManager.java | 6 +++--- .../org/apache/doris/qe/cache/CacheBeProxy.java | 1 + .../org/apache/doris/qe/cache/RowBatchBuilder.java | 14 +++----------- .../java/org/apache/doris/qe/cache/SqlCache.java | 9 --------- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index 74c60c8648db30..726f091a1faab6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -49,7 +49,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; import org.apache.doris.proto.InternalService; -import org.apache.doris.proto.InternalService.PClearType; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.cache.CacheAnalyzer; import org.apache.doris.qe.cache.SqlCache; @@ -362,8 +361,9 @@ private TableIf findTableIf(Env env, FullTableName fullTableName) { return db.get().getTable(fullTableName.table).orElse(null); } - // used in Config.sql_cache_manage_num.callbackClassString and - // Config.cache_last_version_interval_second.callbackClassString, don't remove it + // NOTE: used in Config.sql_cache_manage_num.callbackClassString and + // Config.cache_last_version_interval_second.callbackClassString, + // don't remove it! public static class UpdateConfig extends DefaultConfHandler { @Override public void handle(Field field, String confVal) throws Exception { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java index 70ec56e79dbd55..9be5aacef110d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java @@ -124,6 +124,7 @@ protected boolean clearCache(InternalService.PClearCacheRequest request, Backend backend, int timeoutMs, Status status) { TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort()); try { + request = request.toBuilder().setClearType(InternalService.PClearType.CLEAR_ALL).build(); LOG.info("clear all backend cache, backendId {}", backend.getId()); Future future = BackendServiceProxy.getInstance().clearCache(address, request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java index bfff2a744819ed..5cab6f74bb3411 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Type; import org.apache.doris.proto.InternalService; import org.apache.doris.qe.RowBatch; -import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TResultBatch; import com.google.common.collect.Lists; @@ -131,16 +130,9 @@ public InternalService.PUpdateCacheRequest buildSqlUpdateRequest( .setLastVersion(lastVersion) .setLastVersionTime(lastestTime) .setPartitionNum(partitionNum) - .setCacheOwner(FrontendOptions.getLocalHostAddress()) - .build() - ) - .setDataSize(dataSize) - .addAllRows( - rowList.stream() - .map(row -> ByteString.copyFrom(row)) - .collect(Collectors.toList()) - ) - ).build(); + .build()).setDataSize(dataSize).addAllRows( + rowList.stream().map(row -> ByteString.copyFrom(row)) + .collect(Collectors.toList()))).build(); return updateRequest; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index 247179a062b759..1cbc15452bcdf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -94,15 +94,6 @@ public static Backend findCacheBe(PUniqueId cacheMd5) { return CacheCoordinator.getInstance().findBackend(cacheMd5); } - public static void clearCache(CacheProxy proxy, PUniqueId cacheKeyMd5, PClearType clearType) { - PClearCacheRequest request = PClearCacheRequest.newBuilder() - .setSqlKey(cacheKeyMd5) - .setClearType(clearType) - .build(); - - proxy.clearCache(request); - } - public static InternalService.PFetchCacheResult getCacheData(CacheProxy proxy, PUniqueId cacheKeyMd5, long latestPartitionId, long latestPartitionVersion, long latestPartitionTime, long sumOfPartitionNum, Status status) { From 9cd2e178225b2da631b5347f62ab0237135723d1 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 20:01:52 +0800 Subject: [PATCH 09/20] save2 --- .../org/apache/doris/qe/StmtExecutor.java | 31 ++++++++++++++----- .../apache/doris/qe/cache/CacheAnalyzer.java | 11 ++----- .../org/apache/doris/qe/cache/SqlCache.java | 2 -- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 44770b5bb3ee96..3b341ded8ed810 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -106,6 +106,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.NereidsException; +import org.apache.doris.common.NereidsSqlCacheManager; import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.profile.Profile; @@ -1666,14 +1667,28 @@ private boolean sendCachedValues(MysqlChannel channel, List buildCacheTableList() { } public InternalService.PFetchCacheResult getCacheData() throws UserException { - boolean isNereids = parsedStmt instanceof LogicalPlanAdapter; - if (isNereids) { + if (parsedStmt instanceof LogicalPlanAdapter) { cacheMode = innerCheckCacheModeForNereids(0); } else if (parsedStmt instanceof SelectStmt) { cacheMode = innerCheckCacheMode(0); @@ -527,12 +526,8 @@ public InternalService.PFetchCacheResult getCacheData() throws UserException { return null; } Status status = new Status(); - InternalService.PFetchCacheResult cacheResult = null; - if (!isNereids) { - cacheResult = cache.getCacheData(status); - } - if (!isNereids && status.ok() - && cacheResult != null && cacheResult.getStatus() == InternalService.PCacheStatus.CACHE_OK) { + InternalService.PFetchCacheResult cacheResult = cache.getCacheData(status); + if (status.ok() && cacheResult != null && cacheResult.getStatus() == InternalService.PCacheStatus.CACHE_OK) { int rowCount = 0; int dataSize = 0; for (InternalService.PCacheValue value : cacheResult.getValuesList()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index 1cbc15452bcdf5..0e51ece43b592a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -22,8 +22,6 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.metric.MetricRepo; import org.apache.doris.proto.InternalService; -import org.apache.doris.proto.InternalService.PClearCacheRequest; -import org.apache.doris.proto.InternalService.PClearType; import org.apache.doris.proto.Types.PUniqueId; import org.apache.doris.qe.RowBatch; import org.apache.doris.system.Backend; From 01baa51cb45d53b448da1d8968673ee93fef634e Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 20:04:39 +0800 Subject: [PATCH 10/20] save2 --- .../doris/common/NereidsSqlCacheManager.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index 726f091a1faab6..d2e1634b1c01d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -73,12 +73,7 @@ public class NereidsSqlCacheManager { private volatile Cache sqlCaches; public NereidsSqlCacheManager(int sqlCacheNum, long cacheIntervalSeconds) { - sqlCaches = Caffeine.newBuilder() - .maximumSize(sqlCacheNum) - .expireAfterAccess(Duration.ofSeconds(cacheIntervalSeconds)) - // auto evict cache when jvm memory too low - .softValues() - .build(); + sqlCaches = buildSqlCaches(sqlCacheNum, cacheIntervalSeconds); } public static synchronized void updateConfig() { @@ -91,17 +86,24 @@ public static synchronized void updateConfig() { return; } - int sqlCacheManageNum = Config.sql_cache_manage_num; - int cacheIntervalSecond = Config.cache_last_version_interval_second; + Cache sqlCaches = buildSqlCaches( + Config.sql_cache_manage_num, + Config.cache_last_version_interval_second + ); + sqlCaches.putAll(sqlCacheManager.sqlCaches.asMap()); + sqlCacheManager.sqlCaches = sqlCaches; + } + + private static Cache buildSqlCaches(int sqlCacheNum, long cacheIntervalSeconds) { + sqlCacheNum = sqlCacheNum <= 0 ? 100 : sqlCacheNum; + cacheIntervalSeconds = cacheIntervalSeconds <= 0 ? 30 : cacheIntervalSeconds; - Cache sqlCaches = Caffeine.newBuilder() - .maximumSize(sqlCacheManageNum) - .expireAfterAccess(Duration.ofSeconds(cacheIntervalSecond)) + return Caffeine.newBuilder() + .maximumSize(sqlCacheNum) + .expireAfterAccess(Duration.ofSeconds(cacheIntervalSeconds)) // auto evict cache when jvm memory too low .softValues() .build(); - sqlCaches.putAll(sqlCacheManager.sqlCaches.asMap()); - sqlCacheManager.sqlCaches = sqlCaches; } /** tryAddCache */ From 864337f484fcfe6f424dde9bbcbd6c959fbe77b1 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 20:58:59 +0800 Subject: [PATCH 11/20] save2 --- .../doris/common/NereidsSqlCacheManager.java | 1 - .../apache/doris/nereids/SqlCacheContext.java | 17 +- .../doris/nereids/StatementContext.java | 3 + .../apache/doris/qe/cache/CacheAnalyzer.java | 26 +- .../doris/qe/cache/RowBatchBuilder.java | 5 +- .../org/apache/doris/qe/cache/SqlCache.java | 3 +- .../cache/parse_sql_from_sql_cache.groovy | 1017 +++++++++-------- 7 files changed, 547 insertions(+), 525 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index d2e1634b1c01d1..7f1989e9d7bc7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -123,7 +123,6 @@ public void tryAddCache( if (analyzer.getCache() instanceof SqlCache && (currentMissParseSqlFromSqlCache || sqlCaches.getIfPresent(key) == null)) { SqlCache cache = (SqlCache) analyzer.getCache(); - sqlCacheContext.setCacheKeyMd5(cache.getOrComputeCacheMd5()); sqlCacheContext.setSumOfPartitionNum(cache.getSumOfPartitionNum()); sqlCacheContext.setLatestPartitionId(cache.getLatestId()); sqlCacheContext.setLatestPartitionVersion(cache.getLatestVersion()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index 6afda4e4fe96d6..d5821f98f74c51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -38,6 +38,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Locale; @@ -52,6 +53,7 @@ public class SqlCacheContext { private final TUniqueId queryId; // if contains udf/udaf/tableValuesFunction we can not process it and skip use sql cache private volatile boolean cannotProcessExpression; + private volatile String originSql; private volatile String physicalPlan; private volatile long latestPartitionId = -1; private volatile long latestPartitionTime = -1; @@ -314,8 +316,19 @@ public PUniqueId getCacheKeyMd5() { return cacheKeyMd5; } - public void setCacheKeyMd5(PUniqueId cacheKeyMd5) { - this.cacheKeyMd5 = cacheKeyMd5; + public synchronized PUniqueId getOrComputeCacheKeyMd5() { + if (cacheKeyMd5 == null) { + StringBuilder cacheKey = new StringBuilder(originSql); + if (!usedViews.isEmpty()) { + cacheKey.append(StringUtils.join(usedViews.values(), "|")); + } + cacheKeyMd5 = CacheProxy.getMd5(cacheKey.toString()); + } + return cacheKeyMd5; + } + + public void setOriginSql(String originSql) { + this.originSql = originSql.trim(); } /** FullTableName */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 423d7fd8902fa2..e7bba55be3bf54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -169,6 +169,9 @@ public ConnectContext getConnectContext() { public void setOriginStatement(OriginStatement originStatement) { this.originStatement = originStatement; + if (originStatement != null && sqlCacheContext != null) { + sqlCacheContext.setOriginSql(originStatement.originStmt.trim()); + } } public OriginStatement getOriginStatement() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index f1c5d98f16095f..1993835ad05204 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -45,6 +45,8 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.source.HiveScanNode; import org.apache.doris.metric.MetricRepo; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.SqlCacheContext; import org.apache.doris.nereids.SqlCacheContext.FullTableName; import org.apache.doris.nereids.SqlCacheContext.ScanTable; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -70,6 +72,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; /** @@ -425,12 +428,6 @@ private CacheMode innerCheckCacheModeForNereids(long now) { return CacheMode.NoNeed; } - boolean isNewAllViewExpandStmtListStr = allViewExpandStmtListStr == null; - if (isNewAllViewExpandStmtListStr) { - allViewStmtSet.addAll(((LogicalPlanAdapter) parsedStmt).getViewDdlSqls()); - allViewExpandStmtListStr = StringUtils.join(allViewStmtSet, "|"); - } - if (now == 0) { now = nowtime(); } @@ -442,13 +439,20 @@ private CacheMode innerCheckCacheModeForNereids(long now) { Config.cache_last_version_interval_second * 1000); } + String originStmt = ((LogicalPlanAdapter) parsedStmt).getStatementContext() + .getOriginStatement().originStmt; + cache = new SqlCache(this.queryId, originStmt); + SqlCache sqlCache = (SqlCache) cache; + NereidsPlanner nereidsPlanner = (NereidsPlanner) planner; + Optional sqlCacheContext = nereidsPlanner + .getCascadesContext() + .getStatementContext() + .getSqlCacheContext(); PUniqueId existsMd5 = null; - if (cache instanceof SqlCache) { - existsMd5 = ((SqlCache) cache).getOrComputeCacheMd5(); + if (sqlCacheContext.isPresent()) { + existsMd5 = sqlCacheContext.get().getOrComputeCacheKeyMd5(); } - cache = new SqlCache(this.queryId, ((LogicalPlanAdapter) parsedStmt).getStatementContext() - .getOriginStatement().originStmt); - SqlCache sqlCache = (SqlCache) cache; + sqlCache.setCacheInfo(this.latestTable, allViewExpandStmtListStr); sqlCache.setCacheMd5(existsMd5); MetricRepo.COUNTER_CACHE_ADDED_SQL.increase(1L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java index 5cab6f74bb3411..20d9f93efb494e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Type; import org.apache.doris.proto.InternalService; +import org.apache.doris.proto.Types.PUniqueId; import org.apache.doris.qe.RowBatch; import org.apache.doris.thrift.TResultBatch; @@ -117,10 +118,10 @@ public void clear() { } public InternalService.PUpdateCacheRequest buildSqlUpdateRequest( - String sql, long partitionKey, long lastVersion, long lastestTime, long partitionNum) { + PUniqueId cacheKeyMd5, long partitionKey, long lastVersion, long lastestTime, long partitionNum) { if (updateRequest == null) { updateRequest = InternalService.PUpdateCacheRequest.newBuilder() - .setSqlKey(CacheProxy.getMd5(sql)) + .setSqlKey(cacheKeyMd5) .setCacheType(InternalService.CacheType.SQL_CACHE).build(); } updateRequest = updateRequest.toBuilder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index 0e51ece43b592a..efe4ed5949f6d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -141,8 +141,9 @@ public void updateCache() { return; } + PUniqueId cacheKeyMd5 = getOrComputeCacheMd5(); InternalService.PUpdateCacheRequest updateRequest = - rowBatchBuilder.buildSqlUpdateRequest(getSqlWithViewStmt(), + rowBatchBuilder.buildSqlUpdateRequest(cacheKeyMd5, latestTable.latestPartitionId, latestTable.latestPartitionVersion, latestTable.latestPartitionTime, diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index 0a318218301dfc..ba467d9a9ac326 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -51,514 +51,515 @@ suite("parse_sql_from_sql_cache") { // use sql cache assertHasCache "select * from test_use_plan_cache" - }), - extraThread("testAddPartitionAndInsert", { - createTestTable "test_use_plan_cache2" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache2" - sql "select * from test_use_plan_cache2" - assertHasCache "select * from test_use_plan_cache2" - - // add empty partition can use cache - sql "alter table test_use_plan_cache2 add partition p6 values[('6'),('7'))" - assertHasCache "select * from test_use_plan_cache2" - - // insert data can not use cache - sql "insert into test_use_plan_cache2 values(6, 1)" - assertNoCache "select * from test_use_plan_cache2" - }), - extraThread("testDropPartition", { - createTestTable "test_use_plan_cache3" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache3" - sql "select * from test_use_plan_cache3" - assertHasCache "select * from test_use_plan_cache3" - - // drop partition can not use cache - sql "alter table test_use_plan_cache3 drop partition p5" - assertNoCache "select * from test_use_plan_cache3" - }), - extraThread("testReplacePartition", { - createTestTable "test_use_plan_cache4" - - sql "alter table test_use_plan_cache4 add temporary partition tp1 values [('1'), ('2'))" - - streamLoad { - table "test_use_plan_cache4" - set "temporaryPartitions", "tp1" - inputIterator([[1, 3], [1, 4]].iterator()) - } - sql "sync" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache4" - sql "select * from test_use_plan_cache4" - assertHasCache "select * from test_use_plan_cache4" - - // replace partition can not use cache - sql "alter table test_use_plan_cache4 replace partition (p1) with temporary partition(tp1)" - assertNoCache "select * from test_use_plan_cache4" - }), - extraThread("testStreamLoad", { - createTestTable "test_use_plan_cache5" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache5" - sql "select * from test_use_plan_cache5" - assertHasCache "select * from test_use_plan_cache5" - - streamLoad { - table "test_use_plan_cache5" - set "partitions", "p1" - inputIterator([[1, 3], [1, 4]].iterator()) - } - - // stream load can not use cache - sql "select * from test_use_plan_cache5" - assertNoCache "select * from test_use_plan_cache5" - }), - extraThread("testUpdate",{ - createTestTable "test_use_plan_cache6", uniqueTable=true - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache6" - sql "select * from test_use_plan_cache6" - assertHasCache "select * from test_use_plan_cache6" - - sql "update test_use_plan_cache6 set value=3 where id=1" - - // update can not use cache - sql "select * from test_use_plan_cache6" - assertNoCache "select * from test_use_plan_cache6" - }), - extraThread("testDelete", { - createTestTable "test_use_plan_cache7" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache7" - sql "select * from test_use_plan_cache7" - assertHasCache "select * from test_use_plan_cache7" - - sql "delete from test_use_plan_cache7 where id = 1" - - // delete can not use cache - sql "select * from test_use_plan_cache7" - assertNoCache "select * from test_use_plan_cache7" - }), - extraThread("testDropTable", { - createTestTable "test_use_plan_cache8" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache8" - sql "select * from test_use_plan_cache8" - assertHasCache "select * from test_use_plan_cache8" - - sql "drop table test_use_plan_cache8" - - // should visible the table has bean deleted - test { - sql "select * from test_use_plan_cache8" - exception "does not exist in database" - } - }), - extraThread("testCreateAndAlterView", { - createTestTable "test_use_plan_cache9" - - sql "drop view if exists test_use_plan_cache9_view" - sql "create view test_use_plan_cache9_view as select * from test_use_plan_cache9" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache9_view" - sql "select * from test_use_plan_cache9_view" - assertHasCache "select * from test_use_plan_cache9_view" - - // alter view should not use cache - sql "alter view test_use_plan_cache9_view as select id from test_use_plan_cache9" - assertNoCache "select * from test_use_plan_cache9_view" - }), - extraThread("testDropView", { - createTestTable "test_use_plan_cache10" - - sql "drop view if exists test_use_plan_cache10_view" - sql "create view test_use_plan_cache10_view as select * from test_use_plan_cache10" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache10_view" - sql "select * from test_use_plan_cache10_view" - assertHasCache "select * from test_use_plan_cache10_view" - - sql "drop view test_use_plan_cache10_view" - // should visible the view has bean deleted - test { - sql "select * from test_use_plan_cache10_view" - exception "does not exist in database" - } - }), - extraThread("testBaseTableChanged", { - createTestTable "test_use_plan_cache11" - - sql "drop view if exists test_use_plan_cache11_view" - sql "create view test_use_plan_cache11_view as select * from test_use_plan_cache11" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache11_view" - sql "select * from test_use_plan_cache11_view" - assertHasCache "select * from test_use_plan_cache11_view" - - sql "insert into test_use_plan_cache11 values(1, 3)" - - // base table already changed, can not use cache - assertNoCache "select * from test_use_plan_cache11_view" - }), - extraThread("testNotShareCacheBetweenUsers", { - sql "drop user if exists test_cache_user1" - sql "create user test_cache_user1 identified by 'DORIS@2024'" - def dbName = context.config.getDbNameByFile(context.file) - sql """GRANT SELECT_PRIV ON *.* TO test_cache_user1""" - - createTestTable "test_use_plan_cache12" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache12" - sql "select * from test_use_plan_cache12" - assertHasCache "select * from test_use_plan_cache12" - - sql "sync" - - - extraThread("test_cache_user1_thread", { - connect(user = "test_cache_user1", password="DORIS@2024") { - sql "use ${dbName}" - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache12" - } - }).get() - }), - extraThread("testAddRowPolicy", { - def dbName = context.config.getDbNameByFile(context.file) - sql "set enable_nereids_planner=false" - try_sql """ - DROP ROW POLICY if exists test_cache_row_policy_2 - ON ${dbName}.test_use_plan_cache13 - FOR test_cache_user2""" - sql "set enable_nereids_planner=true" - - sql "drop user if exists test_cache_user2" - sql "create user test_cache_user2 identified by 'DORIS@2024'" - sql """GRANT SELECT_PRIV ON *.* TO test_cache_user2""" - - createTestTable "test_use_plan_cache13" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "sync" - - extraThread("test_cache_user2_thread", { - connect(user = "test_cache_user2", password="DORIS@2024") { - sql "use ${dbName}" - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache13" - sql "select * from test_use_plan_cache13" - assertHasCache "select * from test_use_plan_cache13" - } - }).get() - - sql "set enable_nereids_planner=false" - sql """ - CREATE ROW POLICY test_cache_row_policy_2 - ON ${dbName}.test_use_plan_cache13 - AS RESTRICTIVE TO test_cache_user2 - USING (id = 'concat(id, "**")')""" - sql "set enable_nereids_planner=true" - - sql "sync" - - // after row policy changed, the cache is invalidate - extraThread("test_cache_user2_thread2", { - connect(user = "test_cache_user2", password="DORIS@2024") { - sql "use ${dbName}" - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache13" - } - }).get() - }), - extraThread("testDropRowPolicy", { - def dbName = context.config.getDbNameByFile(context.file) - sql "set enable_nereids_planner=false" - try_sql """ - DROP ROW POLICY if exists test_cache_row_policy_3 - ON ${dbName}.test_use_plan_cache14 - FOR test_cache_user3""" - sql "set enable_nereids_planner=true" - - sql "drop user if exists test_cache_user3" - sql "create user test_cache_user3 identified by 'DORIS@2024'" - sql """GRANT SELECT_PRIV ON *.* TO test_cache_user3""" - - createTestTable "test_use_plan_cache14" - - sql "set enable_nereids_planner=false" - sql """ - CREATE ROW POLICY test_cache_row_policy_3 - ON ${dbName}.test_use_plan_cache14 - AS RESTRICTIVE TO test_cache_user3 - USING (id = 'concat(id, "**")')""" - sql "set enable_nereids_planner=true" - - sql "sync" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - extraThread("test_cache_user3_thread", { - connect(user = "test_cache_user3", password="DORIS@2024") { - sql "use ${dbName}" - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache14" - sql "select * from test_use_plan_cache14" - assertHasCache "select * from test_use_plan_cache14" - } - }).get() - - sql "set enable_nereids_planner=false" - try_sql """ - DROP ROW POLICY if exists test_cache_row_policy_3 - ON ${dbName}.test_use_plan_cache14 - FOR test_cache_user3""" - sql "set enable_nereids_planner=true" - - sql "sync" - - // after row policy changed, the cache is invalidate - extraThread("test_cache_user3_thread2", { - connect(user = "test_cache_user3", password="DORIS@2024") { - sql "use ${dbName}" - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache14" - } - }).get() - }), - extraThread("testRemovePrivilege", { - def dbName = context.config.getDbNameByFile(context.file) - - createTestTable "test_use_plan_cache15" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "drop user if exists test_cache_user4" - sql "create user test_cache_user4 identified by 'DORIS@2024'" - sql "GRANT SELECT_PRIV ON regression_test.* TO test_cache_user4" - sql "GRANT SELECT_PRIV ON ${dbName}.test_use_plan_cache15 TO test_cache_user4" - - sql "sync" - - extraThread("test_cache_user4_thread", { - connect(user = "test_cache_user4", password="DORIS@2024") { - sql "use ${dbName}" - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select * from test_use_plan_cache15" - sql "select * from test_use_plan_cache15" - assertHasCache "select * from test_use_plan_cache15" - } - }).get() - - sql "REVOKE SELECT_PRIV ON ${dbName}.test_use_plan_cache15 FROM test_cache_user4" - - sql "sync" - - // after privileges changed, the cache is invalidate - extraThread("test_cache_user4_thread2", { - connect(user = "test_cache_user4", password="DORIS@2024") { - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - test { - sql ("select * from ${dbName}.test_use_plan_cache15") - exception "Permission denied" - } - } - }).get() - }), - extraThread("testNondeterministic", { - createTestTable "test_use_plan_cache16" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - assertNoCache "select random() from test_use_plan_cache16" - // create sql cache - sql "select random() from test_use_plan_cache16" - // can not use sql cache - assertNoCache "select random() from test_use_plan_cache16" - - - assertNoCache "select year(now()) from test_use_plan_cache16" - sql "select year(now()) from test_use_plan_cache16" - assertHasCache "select year(now()) from test_use_plan_cache16" - - - assertNoCache "select second(now()) from test_use_plan_cache16" - sql "select second(now()) from test_use_plan_cache16" - sleep(1000) - assertNoCache "select second(now()) from test_use_plan_cache16" - }), - extraThread("testUserVariable", { - // make sure if the table has been dropped, the cache should invalidate, - // so we should retry twice to check - for (def i in 0..2) { - createTestTable "test_use_plan_cache17" - - // after partition changed 10s, the sql cache can be used - sleep(10000) - - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_sql_cache=true" - - sql "set @custom_variable=10" - assertNoCache "select @custom_variable from test_use_plan_cache17" - // create sql cache - sql "select @custom_variable from test_use_plan_cache17" - // can use sql cache - assertHasCache "select @custom_variable from test_use_plan_cache17" - - sql "set @custom_variable=20" - assertNoCache "select @custom_variable from test_use_plan_cache17" - } - }), - extraThread("test_udf", { - def jarPath = """${context.config.suitePath}/javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar""" - try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);") - try_sql("DROP TABLE IF EXISTS test_javaudf_string") - - sql """ DROP TABLE IF EXISTS test_javaudf_string """ - sql """ - CREATE TABLE IF NOT EXISTS test_javaudf_string ( - `user_id` INT NOT NULL COMMENT "用户id", - `char_col` CHAR NOT NULL COMMENT "", - `varchar_col` VARCHAR(10) NOT NULL COMMENT "", - `string_col` STRING NOT NULL COMMENT "" - ) - DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); - """ - - StringBuilder values = new StringBuilder() - int i = 1 - for (; i < 9; i ++) { - values.append(" (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),\n") - } - values.append("(${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')") - - sql "INSERT INTO test_javaudf_string VALUES ${values}" - sql "sync" - - File path = new File(jarPath) - if (!path.exists()) { - throw new IllegalStateException("""${jarPath} doesn't exist! """) - } - - sql """ CREATE FUNCTION java_udf_string_test(string, int, int) RETURNS string PROPERTIES ( - "file"="file://${jarPath}", - "symbol"="org.apache.doris.udf.StringTest", - "type"="JAVA_UDF" - ); """ - - assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" - sql "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" - assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" }) +// , +// extraThread("testAddPartitionAndInsert", { +// createTestTable "test_use_plan_cache2" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache2" +// sql "select * from test_use_plan_cache2" +// assertHasCache "select * from test_use_plan_cache2" +// +// // add empty partition can use cache +// sql "alter table test_use_plan_cache2 add partition p6 values[('6'),('7'))" +// assertHasCache "select * from test_use_plan_cache2" +// +// // insert data can not use cache +// sql "insert into test_use_plan_cache2 values(6, 1)" +// assertNoCache "select * from test_use_plan_cache2" +// }), +// extraThread("testDropPartition", { +// createTestTable "test_use_plan_cache3" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache3" +// sql "select * from test_use_plan_cache3" +// assertHasCache "select * from test_use_plan_cache3" +// +// // drop partition can not use cache +// sql "alter table test_use_plan_cache3 drop partition p5" +// assertNoCache "select * from test_use_plan_cache3" +// }), +// extraThread("testReplacePartition", { +// createTestTable "test_use_plan_cache4" +// +// sql "alter table test_use_plan_cache4 add temporary partition tp1 values [('1'), ('2'))" +// +// streamLoad { +// table "test_use_plan_cache4" +// set "temporaryPartitions", "tp1" +// inputIterator([[1, 3], [1, 4]].iterator()) +// } +// sql "sync" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache4" +// sql "select * from test_use_plan_cache4" +// assertHasCache "select * from test_use_plan_cache4" +// +// // replace partition can not use cache +// sql "alter table test_use_plan_cache4 replace partition (p1) with temporary partition(tp1)" +// assertNoCache "select * from test_use_plan_cache4" +// }), +// extraThread("testStreamLoad", { +// createTestTable "test_use_plan_cache5" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache5" +// sql "select * from test_use_plan_cache5" +// assertHasCache "select * from test_use_plan_cache5" +// +// streamLoad { +// table "test_use_plan_cache5" +// set "partitions", "p1" +// inputIterator([[1, 3], [1, 4]].iterator()) +// } +// +// // stream load can not use cache +// sql "select * from test_use_plan_cache5" +// assertNoCache "select * from test_use_plan_cache5" +// }), +// extraThread("testUpdate",{ +// createTestTable "test_use_plan_cache6", uniqueTable=true +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache6" +// sql "select * from test_use_plan_cache6" +// assertHasCache "select * from test_use_plan_cache6" +// +// sql "update test_use_plan_cache6 set value=3 where id=1" +// +// // update can not use cache +// sql "select * from test_use_plan_cache6" +// assertNoCache "select * from test_use_plan_cache6" +// }), +// extraThread("testDelete", { +// createTestTable "test_use_plan_cache7" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache7" +// sql "select * from test_use_plan_cache7" +// assertHasCache "select * from test_use_plan_cache7" +// +// sql "delete from test_use_plan_cache7 where id = 1" +// +// // delete can not use cache +// sql "select * from test_use_plan_cache7" +// assertNoCache "select * from test_use_plan_cache7" +// }), +// extraThread("testDropTable", { +// createTestTable "test_use_plan_cache8" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache8" +// sql "select * from test_use_plan_cache8" +// assertHasCache "select * from test_use_plan_cache8" +// +// sql "drop table test_use_plan_cache8" +// +// // should visible the table has bean deleted +// test { +// sql "select * from test_use_plan_cache8" +// exception "does not exist in database" +// } +// }), +// extraThread("testCreateAndAlterView", { +// createTestTable "test_use_plan_cache9" +// +// sql "drop view if exists test_use_plan_cache9_view" +// sql "create view test_use_plan_cache9_view as select * from test_use_plan_cache9" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache9_view" +// sql "select * from test_use_plan_cache9_view" +// assertHasCache "select * from test_use_plan_cache9_view" +// +// // alter view should not use cache +// sql "alter view test_use_plan_cache9_view as select id from test_use_plan_cache9" +// assertNoCache "select * from test_use_plan_cache9_view" +// }), +// extraThread("testDropView", { +// createTestTable "test_use_plan_cache10" +// +// sql "drop view if exists test_use_plan_cache10_view" +// sql "create view test_use_plan_cache10_view as select * from test_use_plan_cache10" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache10_view" +// sql "select * from test_use_plan_cache10_view" +// assertHasCache "select * from test_use_plan_cache10_view" +// +// sql "drop view test_use_plan_cache10_view" +// // should visible the view has bean deleted +// test { +// sql "select * from test_use_plan_cache10_view" +// exception "does not exist in database" +// } +// }), +// extraThread("testBaseTableChanged", { +// createTestTable "test_use_plan_cache11" +// +// sql "drop view if exists test_use_plan_cache11_view" +// sql "create view test_use_plan_cache11_view as select * from test_use_plan_cache11" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache11_view" +// sql "select * from test_use_plan_cache11_view" +// assertHasCache "select * from test_use_plan_cache11_view" +// +// sql "insert into test_use_plan_cache11 values(1, 3)" +// +// // base table already changed, can not use cache +// assertNoCache "select * from test_use_plan_cache11_view" +// }), +// extraThread("testNotShareCacheBetweenUsers", { +// sql "drop user if exists test_cache_user1" +// sql "create user test_cache_user1 identified by 'DORIS@2024'" +// def dbName = context.config.getDbNameByFile(context.file) +// sql """GRANT SELECT_PRIV ON *.* TO test_cache_user1""" +// +// createTestTable "test_use_plan_cache12" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache12" +// sql "select * from test_use_plan_cache12" +// assertHasCache "select * from test_use_plan_cache12" +// +// sql "sync" +// +// +// extraThread("test_cache_user1_thread", { +// connect(user = "test_cache_user1", password="DORIS@2024") { +// sql "use ${dbName}" +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache12" +// } +// }).get() +// }), +// extraThread("testAddRowPolicy", { +// def dbName = context.config.getDbNameByFile(context.file) +// sql "set enable_nereids_planner=false" +// try_sql """ +// DROP ROW POLICY if exists test_cache_row_policy_2 +// ON ${dbName}.test_use_plan_cache13 +// FOR test_cache_user2""" +// sql "set enable_nereids_planner=true" +// +// sql "drop user if exists test_cache_user2" +// sql "create user test_cache_user2 identified by 'DORIS@2024'" +// sql """GRANT SELECT_PRIV ON *.* TO test_cache_user2""" +// +// createTestTable "test_use_plan_cache13" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "sync" +// +// extraThread("test_cache_user2_thread", { +// connect(user = "test_cache_user2", password="DORIS@2024") { +// sql "use ${dbName}" +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache13" +// sql "select * from test_use_plan_cache13" +// assertHasCache "select * from test_use_plan_cache13" +// } +// }).get() +// +// sql "set enable_nereids_planner=false" +// sql """ +// CREATE ROW POLICY test_cache_row_policy_2 +// ON ${dbName}.test_use_plan_cache13 +// AS RESTRICTIVE TO test_cache_user2 +// USING (id = 'concat(id, "**")')""" +// sql "set enable_nereids_planner=true" +// +// sql "sync" +// +// // after row policy changed, the cache is invalidate +// extraThread("test_cache_user2_thread2", { +// connect(user = "test_cache_user2", password="DORIS@2024") { +// sql "use ${dbName}" +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache13" +// } +// }).get() +// }), +// extraThread("testDropRowPolicy", { +// def dbName = context.config.getDbNameByFile(context.file) +// sql "set enable_nereids_planner=false" +// try_sql """ +// DROP ROW POLICY if exists test_cache_row_policy_3 +// ON ${dbName}.test_use_plan_cache14 +// FOR test_cache_user3""" +// sql "set enable_nereids_planner=true" +// +// sql "drop user if exists test_cache_user3" +// sql "create user test_cache_user3 identified by 'DORIS@2024'" +// sql """GRANT SELECT_PRIV ON *.* TO test_cache_user3""" +// +// createTestTable "test_use_plan_cache14" +// +// sql "set enable_nereids_planner=false" +// sql """ +// CREATE ROW POLICY test_cache_row_policy_3 +// ON ${dbName}.test_use_plan_cache14 +// AS RESTRICTIVE TO test_cache_user3 +// USING (id = 'concat(id, "**")')""" +// sql "set enable_nereids_planner=true" +// +// sql "sync" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// extraThread("test_cache_user3_thread", { +// connect(user = "test_cache_user3", password="DORIS@2024") { +// sql "use ${dbName}" +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache14" +// sql "select * from test_use_plan_cache14" +// assertHasCache "select * from test_use_plan_cache14" +// } +// }).get() +// +// sql "set enable_nereids_planner=false" +// try_sql """ +// DROP ROW POLICY if exists test_cache_row_policy_3 +// ON ${dbName}.test_use_plan_cache14 +// FOR test_cache_user3""" +// sql "set enable_nereids_planner=true" +// +// sql "sync" +// +// // after row policy changed, the cache is invalidate +// extraThread("test_cache_user3_thread2", { +// connect(user = "test_cache_user3", password="DORIS@2024") { +// sql "use ${dbName}" +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache14" +// } +// }).get() +// }), +// extraThread("testRemovePrivilege", { +// def dbName = context.config.getDbNameByFile(context.file) +// +// createTestTable "test_use_plan_cache15" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "drop user if exists test_cache_user4" +// sql "create user test_cache_user4 identified by 'DORIS@2024'" +// sql "GRANT SELECT_PRIV ON regression_test.* TO test_cache_user4" +// sql "GRANT SELECT_PRIV ON ${dbName}.test_use_plan_cache15 TO test_cache_user4" +// +// sql "sync" +// +// extraThread("test_cache_user4_thread", { +// connect(user = "test_cache_user4", password="DORIS@2024") { +// sql "use ${dbName}" +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select * from test_use_plan_cache15" +// sql "select * from test_use_plan_cache15" +// assertHasCache "select * from test_use_plan_cache15" +// } +// }).get() +// +// sql "REVOKE SELECT_PRIV ON ${dbName}.test_use_plan_cache15 FROM test_cache_user4" +// +// sql "sync" +// +// // after privileges changed, the cache is invalidate +// extraThread("test_cache_user4_thread2", { +// connect(user = "test_cache_user4", password="DORIS@2024") { +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// test { +// sql ("select * from ${dbName}.test_use_plan_cache15") +// exception "Permission denied" +// } +// } +// }).get() +// }), +// extraThread("testNondeterministic", { +// createTestTable "test_use_plan_cache16" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// assertNoCache "select random() from test_use_plan_cache16" +// // create sql cache +// sql "select random() from test_use_plan_cache16" +// // can not use sql cache +// assertNoCache "select random() from test_use_plan_cache16" +// +// +// assertNoCache "select year(now()) from test_use_plan_cache16" +// sql "select year(now()) from test_use_plan_cache16" +// assertHasCache "select year(now()) from test_use_plan_cache16" +// +// +// assertNoCache "select second(now()) from test_use_plan_cache16" +// sql "select second(now()) from test_use_plan_cache16" +// sleep(1000) +// assertNoCache "select second(now()) from test_use_plan_cache16" +// }), +// extraThread("testUserVariable", { +// // make sure if the table has been dropped, the cache should invalidate, +// // so we should retry twice to check +// for (def i in 0..2) { +// createTestTable "test_use_plan_cache17" +// +// // after partition changed 10s, the sql cache can be used +// sleep(10000) +// +// sql "set enable_nereids_planner=true" +// sql "set enable_fallback_to_original_planner=false" +// sql "set enable_sql_cache=true" +// +// sql "set @custom_variable=10" +// assertNoCache "select @custom_variable from test_use_plan_cache17" +// // create sql cache +// sql "select @custom_variable from test_use_plan_cache17" +// // can use sql cache +// assertHasCache "select @custom_variable from test_use_plan_cache17" +// +// sql "set @custom_variable=20" +// assertNoCache "select @custom_variable from test_use_plan_cache17" +// } +// }), +// extraThread("test_udf", { +// def jarPath = """${context.config.suitePath}/javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar""" +// try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);") +// try_sql("DROP TABLE IF EXISTS test_javaudf_string") +// +// sql """ DROP TABLE IF EXISTS test_javaudf_string """ +// sql """ +// CREATE TABLE IF NOT EXISTS test_javaudf_string ( +// `user_id` INT NOT NULL COMMENT "用户id", +// `char_col` CHAR NOT NULL COMMENT "", +// `varchar_col` VARCHAR(10) NOT NULL COMMENT "", +// `string_col` STRING NOT NULL COMMENT "" +// ) +// DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); +// """ +// +// StringBuilder values = new StringBuilder() +// int i = 1 +// for (; i < 9; i ++) { +// values.append(" (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),\n") +// } +// values.append("(${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')") +// +// sql "INSERT INTO test_javaudf_string VALUES ${values}" +// sql "sync" +// +// File path = new File(jarPath) +// if (!path.exists()) { +// throw new IllegalStateException("""${jarPath} doesn't exist! """) +// } +// +// sql """ CREATE FUNCTION java_udf_string_test(string, int, int) RETURNS string PROPERTIES ( +// "file"="file://${jarPath}", +// "symbol"="org.apache.doris.udf.StringTest", +// "type"="JAVA_UDF" +// ); """ +// +// assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" +// sql "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" +// assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" +// }) ).get() } From 8f739d4457393614a8980b94c4c201ba1fa778bf Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 21:04:29 +0800 Subject: [PATCH 12/20] save2 --- .../doris/common/NereidsSqlCacheManager.java | 10 +- .../apache/doris/nereids/SqlCacheContext.java | 6 +- .../cache/parse_sql_from_sql_cache.groovy | 1017 ++++++++--------- 3 files changed, 515 insertions(+), 518 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index 7f1989e9d7bc7f..b94663aea4bd38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -49,6 +49,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; import org.apache.doris.proto.InternalService; +import org.apache.doris.proto.Types.PUniqueId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.cache.CacheAnalyzer; import org.apache.doris.qe.cache.SqlCache; @@ -120,8 +121,8 @@ public void tryAddCache( SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get(); UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); String key = currentUserIdentity.toString() + ":" + sql.trim(); - if (analyzer.getCache() instanceof SqlCache - && (currentMissParseSqlFromSqlCache || sqlCaches.getIfPresent(key) == null)) { + if ((currentMissParseSqlFromSqlCache || sqlCaches.getIfPresent(key) == null) && + sqlCacheContext.getOrComputeCacheKeyMd5() != null) { SqlCache cache = (SqlCache) analyzer.getCache(); sqlCacheContext.setSumOfPartitionNum(cache.getSumOfPartitionNum()); sqlCacheContext.setLatestPartitionId(cache.getLatestId()); @@ -178,16 +179,17 @@ public Optional tryParseSql(ConnectContext connectContext, Stri try { Status status = new Status(); + PUniqueId cacheKeyMd5 = sqlCacheContext.getOrComputeCacheKeyMd5(); InternalService.PFetchCacheResult cacheData = SqlCache.getCacheData(sqlCacheContext.getCacheProxy(), - sqlCacheContext.getCacheKeyMd5(), sqlCacheContext.getLatestPartitionId(), + cacheKeyMd5, sqlCacheContext.getLatestPartitionId(), sqlCacheContext.getLatestPartitionVersion(), sqlCacheContext.getLatestPartitionTime(), sqlCacheContext.getSumOfPartitionNum(), status); if (status.ok() && cacheData != null && cacheData.getStatus() == InternalService.PCacheStatus.CACHE_OK) { List cacheValues = cacheData.getValuesList(); String cachedPlan = sqlCacheContext.getPhysicalPlan(); - String backendAddress = SqlCache.findCacheBe(sqlCacheContext.getCacheKeyMd5()).getAddress(); + String backendAddress = SqlCache.findCacheBe(cacheKeyMd5).getAddress(); MetricRepo.COUNTER_CACHE_HIT_SQL.increase(1L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index d5821f98f74c51..789af30d65511f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -312,12 +312,8 @@ public TUniqueId getQueryId() { return queryId; } - public PUniqueId getCacheKeyMd5() { - return cacheKeyMd5; - } - public synchronized PUniqueId getOrComputeCacheKeyMd5() { - if (cacheKeyMd5 == null) { + if (cacheKeyMd5 == null && originSql != null) { StringBuilder cacheKey = new StringBuilder(originSql); if (!usedViews.isEmpty()) { cacheKey.append(StringUtils.join(usedViews.values(), "|")); diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index ba467d9a9ac326..0a318218301dfc 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -51,515 +51,514 @@ suite("parse_sql_from_sql_cache") { // use sql cache assertHasCache "select * from test_use_plan_cache" + }), + extraThread("testAddPartitionAndInsert", { + createTestTable "test_use_plan_cache2" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache2" + sql "select * from test_use_plan_cache2" + assertHasCache "select * from test_use_plan_cache2" + + // add empty partition can use cache + sql "alter table test_use_plan_cache2 add partition p6 values[('6'),('7'))" + assertHasCache "select * from test_use_plan_cache2" + + // insert data can not use cache + sql "insert into test_use_plan_cache2 values(6, 1)" + assertNoCache "select * from test_use_plan_cache2" + }), + extraThread("testDropPartition", { + createTestTable "test_use_plan_cache3" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache3" + sql "select * from test_use_plan_cache3" + assertHasCache "select * from test_use_plan_cache3" + + // drop partition can not use cache + sql "alter table test_use_plan_cache3 drop partition p5" + assertNoCache "select * from test_use_plan_cache3" + }), + extraThread("testReplacePartition", { + createTestTable "test_use_plan_cache4" + + sql "alter table test_use_plan_cache4 add temporary partition tp1 values [('1'), ('2'))" + + streamLoad { + table "test_use_plan_cache4" + set "temporaryPartitions", "tp1" + inputIterator([[1, 3], [1, 4]].iterator()) + } + sql "sync" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache4" + sql "select * from test_use_plan_cache4" + assertHasCache "select * from test_use_plan_cache4" + + // replace partition can not use cache + sql "alter table test_use_plan_cache4 replace partition (p1) with temporary partition(tp1)" + assertNoCache "select * from test_use_plan_cache4" + }), + extraThread("testStreamLoad", { + createTestTable "test_use_plan_cache5" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache5" + sql "select * from test_use_plan_cache5" + assertHasCache "select * from test_use_plan_cache5" + + streamLoad { + table "test_use_plan_cache5" + set "partitions", "p1" + inputIterator([[1, 3], [1, 4]].iterator()) + } + + // stream load can not use cache + sql "select * from test_use_plan_cache5" + assertNoCache "select * from test_use_plan_cache5" + }), + extraThread("testUpdate",{ + createTestTable "test_use_plan_cache6", uniqueTable=true + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache6" + sql "select * from test_use_plan_cache6" + assertHasCache "select * from test_use_plan_cache6" + + sql "update test_use_plan_cache6 set value=3 where id=1" + + // update can not use cache + sql "select * from test_use_plan_cache6" + assertNoCache "select * from test_use_plan_cache6" + }), + extraThread("testDelete", { + createTestTable "test_use_plan_cache7" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache7" + sql "select * from test_use_plan_cache7" + assertHasCache "select * from test_use_plan_cache7" + + sql "delete from test_use_plan_cache7 where id = 1" + + // delete can not use cache + sql "select * from test_use_plan_cache7" + assertNoCache "select * from test_use_plan_cache7" + }), + extraThread("testDropTable", { + createTestTable "test_use_plan_cache8" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache8" + sql "select * from test_use_plan_cache8" + assertHasCache "select * from test_use_plan_cache8" + + sql "drop table test_use_plan_cache8" + + // should visible the table has bean deleted + test { + sql "select * from test_use_plan_cache8" + exception "does not exist in database" + } + }), + extraThread("testCreateAndAlterView", { + createTestTable "test_use_plan_cache9" + + sql "drop view if exists test_use_plan_cache9_view" + sql "create view test_use_plan_cache9_view as select * from test_use_plan_cache9" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache9_view" + sql "select * from test_use_plan_cache9_view" + assertHasCache "select * from test_use_plan_cache9_view" + + // alter view should not use cache + sql "alter view test_use_plan_cache9_view as select id from test_use_plan_cache9" + assertNoCache "select * from test_use_plan_cache9_view" + }), + extraThread("testDropView", { + createTestTable "test_use_plan_cache10" + + sql "drop view if exists test_use_plan_cache10_view" + sql "create view test_use_plan_cache10_view as select * from test_use_plan_cache10" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache10_view" + sql "select * from test_use_plan_cache10_view" + assertHasCache "select * from test_use_plan_cache10_view" + + sql "drop view test_use_plan_cache10_view" + // should visible the view has bean deleted + test { + sql "select * from test_use_plan_cache10_view" + exception "does not exist in database" + } + }), + extraThread("testBaseTableChanged", { + createTestTable "test_use_plan_cache11" + + sql "drop view if exists test_use_plan_cache11_view" + sql "create view test_use_plan_cache11_view as select * from test_use_plan_cache11" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache11_view" + sql "select * from test_use_plan_cache11_view" + assertHasCache "select * from test_use_plan_cache11_view" + + sql "insert into test_use_plan_cache11 values(1, 3)" + + // base table already changed, can not use cache + assertNoCache "select * from test_use_plan_cache11_view" + }), + extraThread("testNotShareCacheBetweenUsers", { + sql "drop user if exists test_cache_user1" + sql "create user test_cache_user1 identified by 'DORIS@2024'" + def dbName = context.config.getDbNameByFile(context.file) + sql """GRANT SELECT_PRIV ON *.* TO test_cache_user1""" + + createTestTable "test_use_plan_cache12" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache12" + sql "select * from test_use_plan_cache12" + assertHasCache "select * from test_use_plan_cache12" + + sql "sync" + + + extraThread("test_cache_user1_thread", { + connect(user = "test_cache_user1", password="DORIS@2024") { + sql "use ${dbName}" + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache12" + } + }).get() + }), + extraThread("testAddRowPolicy", { + def dbName = context.config.getDbNameByFile(context.file) + sql "set enable_nereids_planner=false" + try_sql """ + DROP ROW POLICY if exists test_cache_row_policy_2 + ON ${dbName}.test_use_plan_cache13 + FOR test_cache_user2""" + sql "set enable_nereids_planner=true" + + sql "drop user if exists test_cache_user2" + sql "create user test_cache_user2 identified by 'DORIS@2024'" + sql """GRANT SELECT_PRIV ON *.* TO test_cache_user2""" + + createTestTable "test_use_plan_cache13" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "sync" + + extraThread("test_cache_user2_thread", { + connect(user = "test_cache_user2", password="DORIS@2024") { + sql "use ${dbName}" + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache13" + sql "select * from test_use_plan_cache13" + assertHasCache "select * from test_use_plan_cache13" + } + }).get() + + sql "set enable_nereids_planner=false" + sql """ + CREATE ROW POLICY test_cache_row_policy_2 + ON ${dbName}.test_use_plan_cache13 + AS RESTRICTIVE TO test_cache_user2 + USING (id = 'concat(id, "**")')""" + sql "set enable_nereids_planner=true" + + sql "sync" + + // after row policy changed, the cache is invalidate + extraThread("test_cache_user2_thread2", { + connect(user = "test_cache_user2", password="DORIS@2024") { + sql "use ${dbName}" + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache13" + } + }).get() + }), + extraThread("testDropRowPolicy", { + def dbName = context.config.getDbNameByFile(context.file) + sql "set enable_nereids_planner=false" + try_sql """ + DROP ROW POLICY if exists test_cache_row_policy_3 + ON ${dbName}.test_use_plan_cache14 + FOR test_cache_user3""" + sql "set enable_nereids_planner=true" + + sql "drop user if exists test_cache_user3" + sql "create user test_cache_user3 identified by 'DORIS@2024'" + sql """GRANT SELECT_PRIV ON *.* TO test_cache_user3""" + + createTestTable "test_use_plan_cache14" + + sql "set enable_nereids_planner=false" + sql """ + CREATE ROW POLICY test_cache_row_policy_3 + ON ${dbName}.test_use_plan_cache14 + AS RESTRICTIVE TO test_cache_user3 + USING (id = 'concat(id, "**")')""" + sql "set enable_nereids_planner=true" + + sql "sync" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + extraThread("test_cache_user3_thread", { + connect(user = "test_cache_user3", password="DORIS@2024") { + sql "use ${dbName}" + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache14" + sql "select * from test_use_plan_cache14" + assertHasCache "select * from test_use_plan_cache14" + } + }).get() + + sql "set enable_nereids_planner=false" + try_sql """ + DROP ROW POLICY if exists test_cache_row_policy_3 + ON ${dbName}.test_use_plan_cache14 + FOR test_cache_user3""" + sql "set enable_nereids_planner=true" + + sql "sync" + + // after row policy changed, the cache is invalidate + extraThread("test_cache_user3_thread2", { + connect(user = "test_cache_user3", password="DORIS@2024") { + sql "use ${dbName}" + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache14" + } + }).get() + }), + extraThread("testRemovePrivilege", { + def dbName = context.config.getDbNameByFile(context.file) + + createTestTable "test_use_plan_cache15" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "drop user if exists test_cache_user4" + sql "create user test_cache_user4 identified by 'DORIS@2024'" + sql "GRANT SELECT_PRIV ON regression_test.* TO test_cache_user4" + sql "GRANT SELECT_PRIV ON ${dbName}.test_use_plan_cache15 TO test_cache_user4" + + sql "sync" + + extraThread("test_cache_user4_thread", { + connect(user = "test_cache_user4", password="DORIS@2024") { + sql "use ${dbName}" + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache15" + sql "select * from test_use_plan_cache15" + assertHasCache "select * from test_use_plan_cache15" + } + }).get() + + sql "REVOKE SELECT_PRIV ON ${dbName}.test_use_plan_cache15 FROM test_cache_user4" + + sql "sync" + + // after privileges changed, the cache is invalidate + extraThread("test_cache_user4_thread2", { + connect(user = "test_cache_user4", password="DORIS@2024") { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + test { + sql ("select * from ${dbName}.test_use_plan_cache15") + exception "Permission denied" + } + } + }).get() + }), + extraThread("testNondeterministic", { + createTestTable "test_use_plan_cache16" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select random() from test_use_plan_cache16" + // create sql cache + sql "select random() from test_use_plan_cache16" + // can not use sql cache + assertNoCache "select random() from test_use_plan_cache16" + + + assertNoCache "select year(now()) from test_use_plan_cache16" + sql "select year(now()) from test_use_plan_cache16" + assertHasCache "select year(now()) from test_use_plan_cache16" + + + assertNoCache "select second(now()) from test_use_plan_cache16" + sql "select second(now()) from test_use_plan_cache16" + sleep(1000) + assertNoCache "select second(now()) from test_use_plan_cache16" + }), + extraThread("testUserVariable", { + // make sure if the table has been dropped, the cache should invalidate, + // so we should retry twice to check + for (def i in 0..2) { + createTestTable "test_use_plan_cache17" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + sql "set @custom_variable=10" + assertNoCache "select @custom_variable from test_use_plan_cache17" + // create sql cache + sql "select @custom_variable from test_use_plan_cache17" + // can use sql cache + assertHasCache "select @custom_variable from test_use_plan_cache17" + + sql "set @custom_variable=20" + assertNoCache "select @custom_variable from test_use_plan_cache17" + } + }), + extraThread("test_udf", { + def jarPath = """${context.config.suitePath}/javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar""" + try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);") + try_sql("DROP TABLE IF EXISTS test_javaudf_string") + + sql """ DROP TABLE IF EXISTS test_javaudf_string """ + sql """ + CREATE TABLE IF NOT EXISTS test_javaudf_string ( + `user_id` INT NOT NULL COMMENT "用户id", + `char_col` CHAR NOT NULL COMMENT "", + `varchar_col` VARCHAR(10) NOT NULL COMMENT "", + `string_col` STRING NOT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + + StringBuilder values = new StringBuilder() + int i = 1 + for (; i < 9; i ++) { + values.append(" (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),\n") + } + values.append("(${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')") + + sql "INSERT INTO test_javaudf_string VALUES ${values}" + sql "sync" + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """ CREATE FUNCTION java_udf_string_test(string, int, int) RETURNS string PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.StringTest", + "type"="JAVA_UDF" + ); """ + + assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" + sql "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" + assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" }) -// , -// extraThread("testAddPartitionAndInsert", { -// createTestTable "test_use_plan_cache2" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache2" -// sql "select * from test_use_plan_cache2" -// assertHasCache "select * from test_use_plan_cache2" -// -// // add empty partition can use cache -// sql "alter table test_use_plan_cache2 add partition p6 values[('6'),('7'))" -// assertHasCache "select * from test_use_plan_cache2" -// -// // insert data can not use cache -// sql "insert into test_use_plan_cache2 values(6, 1)" -// assertNoCache "select * from test_use_plan_cache2" -// }), -// extraThread("testDropPartition", { -// createTestTable "test_use_plan_cache3" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache3" -// sql "select * from test_use_plan_cache3" -// assertHasCache "select * from test_use_plan_cache3" -// -// // drop partition can not use cache -// sql "alter table test_use_plan_cache3 drop partition p5" -// assertNoCache "select * from test_use_plan_cache3" -// }), -// extraThread("testReplacePartition", { -// createTestTable "test_use_plan_cache4" -// -// sql "alter table test_use_plan_cache4 add temporary partition tp1 values [('1'), ('2'))" -// -// streamLoad { -// table "test_use_plan_cache4" -// set "temporaryPartitions", "tp1" -// inputIterator([[1, 3], [1, 4]].iterator()) -// } -// sql "sync" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache4" -// sql "select * from test_use_plan_cache4" -// assertHasCache "select * from test_use_plan_cache4" -// -// // replace partition can not use cache -// sql "alter table test_use_plan_cache4 replace partition (p1) with temporary partition(tp1)" -// assertNoCache "select * from test_use_plan_cache4" -// }), -// extraThread("testStreamLoad", { -// createTestTable "test_use_plan_cache5" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache5" -// sql "select * from test_use_plan_cache5" -// assertHasCache "select * from test_use_plan_cache5" -// -// streamLoad { -// table "test_use_plan_cache5" -// set "partitions", "p1" -// inputIterator([[1, 3], [1, 4]].iterator()) -// } -// -// // stream load can not use cache -// sql "select * from test_use_plan_cache5" -// assertNoCache "select * from test_use_plan_cache5" -// }), -// extraThread("testUpdate",{ -// createTestTable "test_use_plan_cache6", uniqueTable=true -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache6" -// sql "select * from test_use_plan_cache6" -// assertHasCache "select * from test_use_plan_cache6" -// -// sql "update test_use_plan_cache6 set value=3 where id=1" -// -// // update can not use cache -// sql "select * from test_use_plan_cache6" -// assertNoCache "select * from test_use_plan_cache6" -// }), -// extraThread("testDelete", { -// createTestTable "test_use_plan_cache7" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache7" -// sql "select * from test_use_plan_cache7" -// assertHasCache "select * from test_use_plan_cache7" -// -// sql "delete from test_use_plan_cache7 where id = 1" -// -// // delete can not use cache -// sql "select * from test_use_plan_cache7" -// assertNoCache "select * from test_use_plan_cache7" -// }), -// extraThread("testDropTable", { -// createTestTable "test_use_plan_cache8" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache8" -// sql "select * from test_use_plan_cache8" -// assertHasCache "select * from test_use_plan_cache8" -// -// sql "drop table test_use_plan_cache8" -// -// // should visible the table has bean deleted -// test { -// sql "select * from test_use_plan_cache8" -// exception "does not exist in database" -// } -// }), -// extraThread("testCreateAndAlterView", { -// createTestTable "test_use_plan_cache9" -// -// sql "drop view if exists test_use_plan_cache9_view" -// sql "create view test_use_plan_cache9_view as select * from test_use_plan_cache9" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache9_view" -// sql "select * from test_use_plan_cache9_view" -// assertHasCache "select * from test_use_plan_cache9_view" -// -// // alter view should not use cache -// sql "alter view test_use_plan_cache9_view as select id from test_use_plan_cache9" -// assertNoCache "select * from test_use_plan_cache9_view" -// }), -// extraThread("testDropView", { -// createTestTable "test_use_plan_cache10" -// -// sql "drop view if exists test_use_plan_cache10_view" -// sql "create view test_use_plan_cache10_view as select * from test_use_plan_cache10" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache10_view" -// sql "select * from test_use_plan_cache10_view" -// assertHasCache "select * from test_use_plan_cache10_view" -// -// sql "drop view test_use_plan_cache10_view" -// // should visible the view has bean deleted -// test { -// sql "select * from test_use_plan_cache10_view" -// exception "does not exist in database" -// } -// }), -// extraThread("testBaseTableChanged", { -// createTestTable "test_use_plan_cache11" -// -// sql "drop view if exists test_use_plan_cache11_view" -// sql "create view test_use_plan_cache11_view as select * from test_use_plan_cache11" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache11_view" -// sql "select * from test_use_plan_cache11_view" -// assertHasCache "select * from test_use_plan_cache11_view" -// -// sql "insert into test_use_plan_cache11 values(1, 3)" -// -// // base table already changed, can not use cache -// assertNoCache "select * from test_use_plan_cache11_view" -// }), -// extraThread("testNotShareCacheBetweenUsers", { -// sql "drop user if exists test_cache_user1" -// sql "create user test_cache_user1 identified by 'DORIS@2024'" -// def dbName = context.config.getDbNameByFile(context.file) -// sql """GRANT SELECT_PRIV ON *.* TO test_cache_user1""" -// -// createTestTable "test_use_plan_cache12" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache12" -// sql "select * from test_use_plan_cache12" -// assertHasCache "select * from test_use_plan_cache12" -// -// sql "sync" -// -// -// extraThread("test_cache_user1_thread", { -// connect(user = "test_cache_user1", password="DORIS@2024") { -// sql "use ${dbName}" -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache12" -// } -// }).get() -// }), -// extraThread("testAddRowPolicy", { -// def dbName = context.config.getDbNameByFile(context.file) -// sql "set enable_nereids_planner=false" -// try_sql """ -// DROP ROW POLICY if exists test_cache_row_policy_2 -// ON ${dbName}.test_use_plan_cache13 -// FOR test_cache_user2""" -// sql "set enable_nereids_planner=true" -// -// sql "drop user if exists test_cache_user2" -// sql "create user test_cache_user2 identified by 'DORIS@2024'" -// sql """GRANT SELECT_PRIV ON *.* TO test_cache_user2""" -// -// createTestTable "test_use_plan_cache13" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "sync" -// -// extraThread("test_cache_user2_thread", { -// connect(user = "test_cache_user2", password="DORIS@2024") { -// sql "use ${dbName}" -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache13" -// sql "select * from test_use_plan_cache13" -// assertHasCache "select * from test_use_plan_cache13" -// } -// }).get() -// -// sql "set enable_nereids_planner=false" -// sql """ -// CREATE ROW POLICY test_cache_row_policy_2 -// ON ${dbName}.test_use_plan_cache13 -// AS RESTRICTIVE TO test_cache_user2 -// USING (id = 'concat(id, "**")')""" -// sql "set enable_nereids_planner=true" -// -// sql "sync" -// -// // after row policy changed, the cache is invalidate -// extraThread("test_cache_user2_thread2", { -// connect(user = "test_cache_user2", password="DORIS@2024") { -// sql "use ${dbName}" -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache13" -// } -// }).get() -// }), -// extraThread("testDropRowPolicy", { -// def dbName = context.config.getDbNameByFile(context.file) -// sql "set enable_nereids_planner=false" -// try_sql """ -// DROP ROW POLICY if exists test_cache_row_policy_3 -// ON ${dbName}.test_use_plan_cache14 -// FOR test_cache_user3""" -// sql "set enable_nereids_planner=true" -// -// sql "drop user if exists test_cache_user3" -// sql "create user test_cache_user3 identified by 'DORIS@2024'" -// sql """GRANT SELECT_PRIV ON *.* TO test_cache_user3""" -// -// createTestTable "test_use_plan_cache14" -// -// sql "set enable_nereids_planner=false" -// sql """ -// CREATE ROW POLICY test_cache_row_policy_3 -// ON ${dbName}.test_use_plan_cache14 -// AS RESTRICTIVE TO test_cache_user3 -// USING (id = 'concat(id, "**")')""" -// sql "set enable_nereids_planner=true" -// -// sql "sync" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// extraThread("test_cache_user3_thread", { -// connect(user = "test_cache_user3", password="DORIS@2024") { -// sql "use ${dbName}" -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache14" -// sql "select * from test_use_plan_cache14" -// assertHasCache "select * from test_use_plan_cache14" -// } -// }).get() -// -// sql "set enable_nereids_planner=false" -// try_sql """ -// DROP ROW POLICY if exists test_cache_row_policy_3 -// ON ${dbName}.test_use_plan_cache14 -// FOR test_cache_user3""" -// sql "set enable_nereids_planner=true" -// -// sql "sync" -// -// // after row policy changed, the cache is invalidate -// extraThread("test_cache_user3_thread2", { -// connect(user = "test_cache_user3", password="DORIS@2024") { -// sql "use ${dbName}" -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache14" -// } -// }).get() -// }), -// extraThread("testRemovePrivilege", { -// def dbName = context.config.getDbNameByFile(context.file) -// -// createTestTable "test_use_plan_cache15" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "drop user if exists test_cache_user4" -// sql "create user test_cache_user4 identified by 'DORIS@2024'" -// sql "GRANT SELECT_PRIV ON regression_test.* TO test_cache_user4" -// sql "GRANT SELECT_PRIV ON ${dbName}.test_use_plan_cache15 TO test_cache_user4" -// -// sql "sync" -// -// extraThread("test_cache_user4_thread", { -// connect(user = "test_cache_user4", password="DORIS@2024") { -// sql "use ${dbName}" -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select * from test_use_plan_cache15" -// sql "select * from test_use_plan_cache15" -// assertHasCache "select * from test_use_plan_cache15" -// } -// }).get() -// -// sql "REVOKE SELECT_PRIV ON ${dbName}.test_use_plan_cache15 FROM test_cache_user4" -// -// sql "sync" -// -// // after privileges changed, the cache is invalidate -// extraThread("test_cache_user4_thread2", { -// connect(user = "test_cache_user4", password="DORIS@2024") { -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// test { -// sql ("select * from ${dbName}.test_use_plan_cache15") -// exception "Permission denied" -// } -// } -// }).get() -// }), -// extraThread("testNondeterministic", { -// createTestTable "test_use_plan_cache16" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// assertNoCache "select random() from test_use_plan_cache16" -// // create sql cache -// sql "select random() from test_use_plan_cache16" -// // can not use sql cache -// assertNoCache "select random() from test_use_plan_cache16" -// -// -// assertNoCache "select year(now()) from test_use_plan_cache16" -// sql "select year(now()) from test_use_plan_cache16" -// assertHasCache "select year(now()) from test_use_plan_cache16" -// -// -// assertNoCache "select second(now()) from test_use_plan_cache16" -// sql "select second(now()) from test_use_plan_cache16" -// sleep(1000) -// assertNoCache "select second(now()) from test_use_plan_cache16" -// }), -// extraThread("testUserVariable", { -// // make sure if the table has been dropped, the cache should invalidate, -// // so we should retry twice to check -// for (def i in 0..2) { -// createTestTable "test_use_plan_cache17" -// -// // after partition changed 10s, the sql cache can be used -// sleep(10000) -// -// sql "set enable_nereids_planner=true" -// sql "set enable_fallback_to_original_planner=false" -// sql "set enable_sql_cache=true" -// -// sql "set @custom_variable=10" -// assertNoCache "select @custom_variable from test_use_plan_cache17" -// // create sql cache -// sql "select @custom_variable from test_use_plan_cache17" -// // can use sql cache -// assertHasCache "select @custom_variable from test_use_plan_cache17" -// -// sql "set @custom_variable=20" -// assertNoCache "select @custom_variable from test_use_plan_cache17" -// } -// }), -// extraThread("test_udf", { -// def jarPath = """${context.config.suitePath}/javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar""" -// try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);") -// try_sql("DROP TABLE IF EXISTS test_javaudf_string") -// -// sql """ DROP TABLE IF EXISTS test_javaudf_string """ -// sql """ -// CREATE TABLE IF NOT EXISTS test_javaudf_string ( -// `user_id` INT NOT NULL COMMENT "用户id", -// `char_col` CHAR NOT NULL COMMENT "", -// `varchar_col` VARCHAR(10) NOT NULL COMMENT "", -// `string_col` STRING NOT NULL COMMENT "" -// ) -// DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); -// """ -// -// StringBuilder values = new StringBuilder() -// int i = 1 -// for (; i < 9; i ++) { -// values.append(" (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),\n") -// } -// values.append("(${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')") -// -// sql "INSERT INTO test_javaudf_string VALUES ${values}" -// sql "sync" -// -// File path = new File(jarPath) -// if (!path.exists()) { -// throw new IllegalStateException("""${jarPath} doesn't exist! """) -// } -// -// sql """ CREATE FUNCTION java_udf_string_test(string, int, int) RETURNS string PROPERTIES ( -// "file"="file://${jarPath}", -// "symbol"="org.apache.doris.udf.StringTest", -// "type"="JAVA_UDF" -// ); """ -// -// assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" -// sql "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" -// assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" -// }) ).get() } From abeefce74b752edae3771d97cab93c75750e9ee2 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 21:07:25 +0800 Subject: [PATCH 13/20] save2 --- .../main/java/org/apache/doris/nereids/SqlCacheContext.java | 1 + .../main/java/org/apache/doris/nereids/StatementContext.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index 789af30d65511f..afbcee3720594f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -312,6 +312,7 @@ public TUniqueId getQueryId() { return queryId; } + /** getOrComputeCacheKeyMd5 */ public synchronized PUniqueId getOrComputeCacheKeyMd5() { if (cacheKeyMd5 == null && originSql != null) { StringBuilder cacheKey = new StringBuilder(originSql); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index e7bba55be3bf54..a468a0f0cb7f21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -154,6 +154,9 @@ public StatementContext(ConnectContext connectContext, OriginStatement originSta && CacheAnalyzer.canUseSqlCache(connectContext.getSessionVariable())) { this.sqlCacheContext = new SqlCacheContext( connectContext.getCurrentUserIdentity(), connectContext.queryId()); + if (originStatement != null) { + this.sqlCacheContext.setOriginSql(originStatement.originStmt.trim()); + } } else { this.sqlCacheContext = null; } From 9e965a8f3f19c8b100347514a08077f5a490e10b Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 21:08:14 +0800 Subject: [PATCH 14/20] save2 --- .../java/org/apache/doris/common/NereidsSqlCacheManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index b94663aea4bd38..e332a21599bfd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -121,8 +121,8 @@ public void tryAddCache( SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get(); UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); String key = currentUserIdentity.toString() + ":" + sql.trim(); - if ((currentMissParseSqlFromSqlCache || sqlCaches.getIfPresent(key) == null) && - sqlCacheContext.getOrComputeCacheKeyMd5() != null) { + if ((currentMissParseSqlFromSqlCache || sqlCaches.getIfPresent(key) == null) + && sqlCacheContext.getOrComputeCacheKeyMd5() != null) { SqlCache cache = (SqlCache) analyzer.getCache(); sqlCacheContext.setSumOfPartitionNum(cache.getSumOfPartitionNum()); sqlCacheContext.setLatestPartitionId(cache.getLatestId()); From 81ba16e9d3d18c833e129752860d4c846c792d38 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 21:56:47 +0800 Subject: [PATCH 15/20] save2 --- .../doris/common/NereidsSqlCacheManager.java | 3 +- .../apache/doris/nereids/SqlCacheContext.java | 87 +++++++++++++++++-- .../rules/analysis/ExpressionAnalyzer.java | 8 +- 3 files changed, 87 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index e332a21599bfd0..ad047a1693eb7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -328,7 +328,8 @@ private boolean nondeterministicFunctionChanged( return true; } - List> nondeterministicFunctions = sqlCacheContext.getFoldNondeterministicPairs(); + List> nondeterministicFunctions + = sqlCacheContext.getFoldFullNondeterministicPairs(); if (nondeterministicFunctions.isEmpty()) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index afbcee3720594f..c2f7da8a7d44db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -38,11 +38,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.commons.lang3.StringUtils; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -67,9 +68,13 @@ public class SqlCacheContext { private final Map> rowPolicies = Maps.newLinkedHashMap(); private final Map> dataMaskPolicies = Maps.newLinkedHashMap(); private final Set usedVariables = Sets.newLinkedHashSet(); - // key: the expression which contains nondeterministic function, e.g. date(now()) - // value: the expression which already try to fold nondeterministic function, e.g. '2024-01-01' + // key: the expression which **contains** nondeterministic function, e.g. date_add(date_column, date(now())) + // value: the expression which already try to fold nondeterministic function, + // e.g. date_add(date_column, '2024-01-01') // note that value maybe contains nondeterministic function too, when fold failed + private final List> foldFullNondeterministicPairs = Lists.newArrayList(); + // key: the expression which **is** nondeterministic function, e.g. now() + // value: the expression which already try to fold nondeterministic function, e.g. '2024-01-01 10:01:03' private final List> foldNondeterministicPairs = Lists.newArrayList(); private volatile boolean hasUnsupportedTables; private final List scanTables = Lists.newArrayList(); @@ -208,6 +213,14 @@ public synchronized List getUsedVariables() { return ImmutableList.copyOf(usedVariables); } + public synchronized void addFoldFullNondeterministicPair(Expression unfold, Expression fold) { + foldFullNondeterministicPairs.add(Pair.of(unfold, fold)); + } + + public synchronized List> getFoldFullNondeterministicPairs() { + return ImmutableList.copyOf(foldFullNondeterministicPairs); + } + public synchronized void addFoldNondeterministicPair(Expression unfold, Expression fold) { foldNondeterministicPairs.add(Pair.of(unfold, fold)); } @@ -280,10 +293,6 @@ public synchronized Map> getRowPolicies() { return ImmutableMap.copyOf(rowPolicies); } - public boolean isHasUnsupportedTables() { - return hasUnsupportedTables; - } - public synchronized void addScanTable(ScanTable scanTable) { this.scanTables.add(scanTable); } @@ -317,7 +326,69 @@ public synchronized PUniqueId getOrComputeCacheKeyMd5() { if (cacheKeyMd5 == null && originSql != null) { StringBuilder cacheKey = new StringBuilder(originSql); if (!usedViews.isEmpty()) { - cacheKey.append(StringUtils.join(usedViews.values(), "|")); + cacheKey.append("|"); + + Iterator> it = usedViews.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + cacheKey.append(entry.getKey()).append("=").append(entry.getValue()); + if (it.hasNext()) { + cacheKey.append("|"); + } + } + } + if (!usedVariables.isEmpty()) { + cacheKey.append("|"); + + Iterator it = usedVariables.iterator(); + while (it.hasNext()) { + Variable usedVariable = it.next(); + cacheKey.append(usedVariable.getType().name()) + .append(":").append(usedVariable.getName()) + .append(usedVariable.getRealExpression().toSql()); + if (it.hasNext()) { + cacheKey.append("|"); + } + } + } + if (!foldNondeterministicPairs.isEmpty()) { + cacheKey.append("|"); + Iterator> it = foldNondeterministicPairs.iterator(); + while (it.hasNext()) { + Pair pair = it.next(); + cacheKey.append(pair.key().toSql()).append("=") + .append(pair.value().toSql()); + if (it.hasNext()) { + cacheKey.append("|"); + } + } + } + if (!rowPolicies.isEmpty()) { + cacheKey.append("|"); + Iterator>> it = rowPolicies.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + cacheKey.append(entry.getKey()) + .append("=") + .append(entry.getValue()); + if (it.hasNext()) { + cacheKey.append("|"); + } + } + } + if (!dataMaskPolicies.isEmpty()) { + cacheKey.append("|"); + Iterator>> it + = dataMaskPolicies.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + cacheKey.append(entry.getKey()) + .append("=") + .append(entry.getValue().map(Object::toString).orElse("")); + if (it.hasNext()) { + cacheKey.append("|"); + } + } } cacheKeyMd5 = CacheProxy.getMd5(cacheKey.toString()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index cc85bc01323c75..691e4aca1a337d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -154,14 +154,18 @@ public Expression analyze(Expression expression, ExpressionRewriteContext contex @Override public Expression visitBoundFunction(BoundFunction boundFunction, ExpressionRewriteContext context) { Expression fold = super.visitBoundFunction(boundFunction, context); - if (fold instanceof Nondeterministic) { + boolean unfold = fold instanceof Nondeterministic; + if (unfold) { sqlCacheContext.setCannotProcessExpression(true); } + if (boundFunction instanceof Nondeterministic && !unfold) { + sqlCacheContext.addFoldNondeterministicPair(boundFunction, fold); + } return fold; } }.rewrite(analyzeResult, context); - sqlCacheContext.addFoldNondeterministicPair(analyzeResult, foldNondeterministic); + sqlCacheContext.addFoldFullNondeterministicPair(analyzeResult, foldNondeterministic); return foldNondeterministic; } return analyzeResult; From c969af4072a99d696f08316c955e23a9a08552bb Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 15 Apr 2024 22:24:10 +0800 Subject: [PATCH 16/20] save2 --- .../apache/doris/nereids/SqlCacheContext.java | 120 +++++++++--------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index c2f7da8a7d44db..a8548377a3fda6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -322,75 +322,81 @@ public TUniqueId getQueryId() { } /** getOrComputeCacheKeyMd5 */ - public synchronized PUniqueId getOrComputeCacheKeyMd5() { + public PUniqueId getOrComputeCacheKeyMd5() { if (cacheKeyMd5 == null && originSql != null) { - StringBuilder cacheKey = new StringBuilder(originSql); - if (!usedViews.isEmpty()) { - cacheKey.append("|"); - - Iterator> it = usedViews.entrySet().iterator(); - while (it.hasNext()) { - Entry entry = it.next(); - cacheKey.append(entry.getKey()).append("=").append(entry.getValue()); - if (it.hasNext()) { - cacheKey.append("|"); + synchronized (this) { + if (cacheKeyMd5 != null) { + return cacheKeyMd5; + } + + StringBuilder cacheKey = new StringBuilder(originSql); + if (!usedViews.isEmpty()) { + cacheKey.append("|"); + + Iterator> it = usedViews.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + cacheKey.append(entry.getKey()).append("=").append(entry.getValue()); + if (it.hasNext()) { + cacheKey.append("|"); + } } } - } - if (!usedVariables.isEmpty()) { - cacheKey.append("|"); - - Iterator it = usedVariables.iterator(); - while (it.hasNext()) { - Variable usedVariable = it.next(); - cacheKey.append(usedVariable.getType().name()) - .append(":").append(usedVariable.getName()) - .append(usedVariable.getRealExpression().toSql()); - if (it.hasNext()) { - cacheKey.append("|"); + if (!usedVariables.isEmpty()) { + cacheKey.append("|"); + + Iterator it = usedVariables.iterator(); + while (it.hasNext()) { + Variable usedVariable = it.next(); + cacheKey.append(usedVariable.getType().name()) + .append(":").append(usedVariable.getName()) + .append(usedVariable.getRealExpression().toSql()); + if (it.hasNext()) { + cacheKey.append("|"); + } } } - } - if (!foldNondeterministicPairs.isEmpty()) { - cacheKey.append("|"); - Iterator> it = foldNondeterministicPairs.iterator(); - while (it.hasNext()) { - Pair pair = it.next(); - cacheKey.append(pair.key().toSql()).append("=") - .append(pair.value().toSql()); - if (it.hasNext()) { - cacheKey.append("|"); + if (!foldNondeterministicPairs.isEmpty()) { + cacheKey.append("|"); + Iterator> it = foldNondeterministicPairs.iterator(); + while (it.hasNext()) { + Pair pair = it.next(); + cacheKey.append(pair.key().toSql()).append("=") + .append(pair.value().toSql()); + if (it.hasNext()) { + cacheKey.append("|"); + } } } - } - if (!rowPolicies.isEmpty()) { - cacheKey.append("|"); - Iterator>> it = rowPolicies.entrySet().iterator(); - while (it.hasNext()) { - Entry> entry = it.next(); - cacheKey.append(entry.getKey()) - .append("=") - .append(entry.getValue()); - if (it.hasNext()) { - cacheKey.append("|"); + if (!rowPolicies.isEmpty()) { + cacheKey.append("|"); + Iterator>> it = rowPolicies.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + cacheKey.append(entry.getKey()) + .append("=") + .append(entry.getValue()); + if (it.hasNext()) { + cacheKey.append("|"); + } } } - } - if (!dataMaskPolicies.isEmpty()) { - cacheKey.append("|"); - Iterator>> it - = dataMaskPolicies.entrySet().iterator(); - while (it.hasNext()) { - Entry> entry = it.next(); - cacheKey.append(entry.getKey()) - .append("=") - .append(entry.getValue().map(Object::toString).orElse("")); - if (it.hasNext()) { - cacheKey.append("|"); + if (!dataMaskPolicies.isEmpty()) { + cacheKey.append("|"); + Iterator>> it + = dataMaskPolicies.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + cacheKey.append(entry.getKey()) + .append("=") + .append(entry.getValue().map(Object::toString).orElse("")); + if (it.hasNext()) { + cacheKey.append("|"); + } } } + cacheKeyMd5 = CacheProxy.getMd5(cacheKey.toString()); } - cacheKeyMd5 = CacheProxy.getMd5(cacheKey.toString()); } return cacheKeyMd5; } From f776d304c0dbb003502b54273a8cda8ec699dac3 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 16 Apr 2024 11:12:07 +0800 Subject: [PATCH 17/20] save2 --- .../org/apache/doris/catalog/ScalarType.java | 12 ++++++++---- .../apache/doris/nereids/util/JoinUtils.java | 17 +++++++++++++---- .../java/org/apache/doris/qe/Coordinator.java | 18 +++++++++++++----- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java index 67346fc21609c1..adf064c0000324 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -744,15 +744,19 @@ public void toThrift(TTypeDesc container) { case DECIMAL128: case DECIMAL256: case DATETIMEV2: { - Preconditions.checkArgument(precision >= scale, - String.format("given precision %d is out of scale bound %d", precision, scale)); + if (precision < scale) { + throw new IllegalArgumentException( + String.format("given precision %d is out of scale bound %d", precision, scale)); + } scalarType.setScale(scale); scalarType.setPrecision(precision); break; } case TIMEV2: { - Preconditions.checkArgument(precision >= scale, - String.format("given precision %d is out of scale bound %d", precision, scale)); + if (precision < scale) { + throw new IllegalArgumentException( + String.format("given precision %d is out of scale bound %d", precision, scale)); + } scalarType.setScale(scale); scalarType.setPrecision(precision); break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 5c01fd4df9a87a..79462a9d8e7ebf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -141,11 +141,20 @@ public boolean isHashJoinCondition(EqualPredicate equal) { public static Pair, List> extractExpressionForHashTable(List leftSlots, List rightSlots, List onConditions) { JoinSlotCoverageChecker checker = new JoinSlotCoverageChecker(leftSlots, rightSlots); - Map> mapper = onConditions.stream().collect(Collectors.groupingBy( - expr -> (expr instanceof EqualPredicate) && checker.isHashJoinCondition((EqualPredicate) expr))); + + ImmutableList.Builder hashConditions = ImmutableList.builderWithExpectedSize(onConditions.size()); + ImmutableList.Builder otherConditions = ImmutableList.builderWithExpectedSize(onConditions.size()); + for (Expression expr : onConditions) { + if (expr instanceof EqualPredicate && checker.isHashJoinCondition((EqualPredicate) expr)) { + hashConditions.add(expr); + } else { + otherConditions.add(expr); + } + } + return Pair.of( - mapper.getOrDefault(true, ImmutableList.of()), - mapper.getOrDefault(false, ImmutableList.of()) + hashConditions.build(), + otherConditions.build() ); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index e5b2e1b2479d73..4c2c866578f5c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -62,6 +62,7 @@ import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.planner.ScanNode; import org.apache.doris.planner.SetOperationNode; +import org.apache.doris.planner.SortNode; import org.apache.doris.planner.UnionNode; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PExecPlanFragmentResult; @@ -999,7 +1000,9 @@ private void sendPipelineCtx() throws TException, RpcException, UserException { // 4. send and wait fragments rpc // 4.1 serialize fragment // unsetFields() must be called serially. - beToPipelineExecCtxs.values().stream().forEach(ctxs -> ctxs.unsetFields()); + for (PipelineExecContexts ctxs : beToPipelineExecCtxs.values()) { + ctxs.unsetFields(); + } // serializeFragments() can be called in parallel. final AtomicLong compressedSize = new AtomicLong(0); beToPipelineExecCtxs.values().parallelStream().forEach(ctxs -> { @@ -3809,10 +3812,15 @@ Map toTPipelineParams(int backendNum) int rate = Math.min(Config.query_colocate_join_memory_limit_penalty_factor, instanceExecParams.size()); memLimit = queryOptions.getMemLimit() / rate; } - Set topnFilterSources = scanNodes.stream() - .filter(scanNode -> scanNode instanceof OlapScanNode) - .flatMap(scanNode -> ((OlapScanNode) scanNode).getTopnFilterSortNodes().stream()) - .map(sort -> sort.getId().asInt()).collect(Collectors.toSet()); + Set topnFilterSources = Sets.newLinkedHashSet(); + for (ScanNode scanNode : scanNodes) { + if (scanNode instanceof OlapScanNode) { + for (SortNode sortNode : ((OlapScanNode) scanNode).getTopnFilterSortNodes()) { + topnFilterSources.add(sortNode.getId().asInt()); + } + } + } + Map res = new HashMap(); Map instanceIdx = new HashMap(); TPlanFragment fragmentThrift = fragment.toThrift(); From 48eca6b919f081e61a8482936df69cca38142c8e Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 16 Apr 2024 11:22:10 +0800 Subject: [PATCH 18/20] save2 --- .../org/apache/doris/qe/SessionVariable.java | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index eb1d511f45e01e..c401b6a722a608 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -42,6 +42,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -1639,16 +1640,33 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { public static final String IGNORE_RUNTIME_FILTER_IDS = "ignore_runtime_filter_ids"; public Set getIgnoredRuntimeFilterIds() { - return Arrays.stream(ignoreRuntimeFilterIds.split(",[\\s]*")) - .map(v -> { - int res = -1; + Set ids = Sets.newLinkedHashSet(); + if (ignoreRuntimeFilterIds.isEmpty()) { + return ImmutableSet.of(); + } + for (String v : ignoreRuntimeFilterIds.split(",[\\s]*")) { + int res = -1; + if (!v.isEmpty()) { + boolean isNumber = true; + for (int i = 0; i < v.length(); ++i) { + char c = v.charAt(i); + if (c < '0' || c > '9') { + isNumber = false; + break; + } + } + if (isNumber) { try { - res = Integer.valueOf(v); - } catch (Exception e) { - //ignore it + res = Integer.parseInt(v); + } catch (Throwable t){ + // ignore } - return res; - }).collect(ImmutableSet.toImmutableSet()); + } + + } + ids.add(res); + } + return ids; } public void setIgnoreRuntimeFilterIds(String ignoreRuntimeFilterIds) { From 0fc57eabf4499e21a83abc10da37ce412ab2a38b Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 16 Apr 2024 18:38:58 +0800 Subject: [PATCH 19/20] save2 --- .../apache/doris/nereids/SqlCacheContext.java | 101 ++++++++---------- .../org/apache/doris/qe/SessionVariable.java | 2 +- .../apache/doris/qe/cache/CacheAnalyzer.java | 2 +- .../cache/parse_sql_from_sql_cache.groovy | 63 ++++++++++- 4 files changed, 105 insertions(+), 63 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index a8548377a3fda6..6a7dbd2ed9de4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -39,7 +39,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -330,70 +329,44 @@ public PUniqueId getOrComputeCacheKeyMd5() { } StringBuilder cacheKey = new StringBuilder(originSql); - if (!usedViews.isEmpty()) { - cacheKey.append("|"); - - Iterator> it = usedViews.entrySet().iterator(); - while (it.hasNext()) { - Entry entry = it.next(); - cacheKey.append(entry.getKey()).append("=").append(entry.getValue()); - if (it.hasNext()) { - cacheKey.append("|"); - } - } + for (Entry entry : usedViews.entrySet()) { + cacheKey.append("|") + .append(entry.getKey()) + .append("=") + .append(entry.getValue()); } - if (!usedVariables.isEmpty()) { - cacheKey.append("|"); - - Iterator it = usedVariables.iterator(); - while (it.hasNext()) { - Variable usedVariable = it.next(); - cacheKey.append(usedVariable.getType().name()) - .append(":").append(usedVariable.getName()) - .append(usedVariable.getRealExpression().toSql()); - if (it.hasNext()) { - cacheKey.append("|"); - } - } + for (Variable usedVariable : usedVariables) { + cacheKey.append("|") + .append(usedVariable.getType().name()) + .append(":") + .append(usedVariable.getName()) + .append("=") + .append(usedVariable.getRealExpression().toSql()); } - if (!foldNondeterministicPairs.isEmpty()) { - cacheKey.append("|"); - Iterator> it = foldNondeterministicPairs.iterator(); - while (it.hasNext()) { - Pair pair = it.next(); - cacheKey.append(pair.key().toSql()).append("=") - .append(pair.value().toSql()); - if (it.hasNext()) { - cacheKey.append("|"); - } - } + for (Pair pair : foldNondeterministicPairs) { + cacheKey.append("|") + .append(pair.key().toSql()) + .append("=") + .append(pair.value().toSql()); } - if (!rowPolicies.isEmpty()) { - cacheKey.append("|"); - Iterator>> it = rowPolicies.entrySet().iterator(); - while (it.hasNext()) { - Entry> entry = it.next(); - cacheKey.append(entry.getKey()) - .append("=") - .append(entry.getValue()); - if (it.hasNext()) { - cacheKey.append("|"); - } + for (Entry> entry : rowPolicies.entrySet()) { + List policy = entry.getValue(); + if (policy.isEmpty()) { + continue; } + cacheKey.append("|") + .append(entry.getKey()) + .append("=") + .append(policy); } - if (!dataMaskPolicies.isEmpty()) { - cacheKey.append("|"); - Iterator>> it - = dataMaskPolicies.entrySet().iterator(); - while (it.hasNext()) { - Entry> entry = it.next(); - cacheKey.append(entry.getKey()) - .append("=") - .append(entry.getValue().map(Object::toString).orElse("")); - if (it.hasNext()) { - cacheKey.append("|"); - } + for (Entry> entry : dataMaskPolicies.entrySet()) { + if (!entry.getValue().isPresent()) { + continue; } + cacheKey.append("|") + .append(entry.getKey()) + .append("=") + .append(entry.getValue().map(Object::toString).orElse("")); } cacheKeyMd5 = CacheProxy.getMd5(cacheKey.toString()); } @@ -412,6 +385,11 @@ public static class FullTableName { public final String catalog; public final String db; public final String table; + + @Override + public String toString() { + return catalog + "." + db + "." + table; + } } /** FullColumnName */ @@ -422,6 +400,11 @@ public static class FullColumnName { public final String db; public final String table; public final String column; + + @Override + public String toString() { + return catalog + "." + db + "." + table + "." + column; + } } /** ScanTable */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c401b6a722a608..064243c62692c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1658,7 +1658,7 @@ public Set getIgnoredRuntimeFilterIds() { if (isNumber) { try { res = Integer.parseInt(v); - } catch (Throwable t){ + } catch (Throwable t) { // ignore } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index 1993835ad05204..a84c4d68c745e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -404,7 +404,7 @@ private CacheMode innerCheckCacheModeForNereids(long now) { } return CacheMode.NoNeed; } - if (!(parsedStmt instanceof LogicalPlanAdapter) || scanNodes.size() == 0) { + if (!(parsedStmt instanceof LogicalPlanAdapter) || scanNodes.isEmpty()) { if (LOG.isDebugEnabled()) { LOG.debug("not a select stmt or no scan node. queryid {}", DebugUtil.printId(queryId)); } diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index 0a318218301dfc..e73498442ae48b 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +import java.util.stream.Collectors + suite("parse_sql_from_sql_cache") { def assertHasCache = { String sqlStr -> explain { @@ -343,7 +345,7 @@ suite("parse_sql_from_sql_cache") { CREATE ROW POLICY test_cache_row_policy_2 ON ${dbName}.test_use_plan_cache13 AS RESTRICTIVE TO test_cache_user2 - USING (id = 'concat(id, "**")')""" + USING (id = 4)""" sql "set enable_nereids_planner=true" sql "sync" @@ -380,7 +382,7 @@ suite("parse_sql_from_sql_cache") { CREATE ROW POLICY test_cache_row_policy_3 ON ${dbName}.test_use_plan_cache14 AS RESTRICTIVE TO test_cache_user3 - USING (id = 'concat(id, "**")')""" + USING (id = 4)""" sql "set enable_nereids_planner=true" sql "sync" @@ -559,6 +561,63 @@ suite("parse_sql_from_sql_cache") { assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" sql "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;" + }), + extraThread("testMultiFrontends", { + def aliveFrontends = sql_return_maparray("show frontends") + .stream() + .filter { it["Alive"].toString().equalsIgnoreCase("true") } + .collect(Collectors.toList()) + + if (aliveFrontends.size() <= 1) { + return + } + + def fe1 = aliveFrontends[0]["Host"] + ":" + aliveFrontends[0]["QueryPort"] + def fe2 = fe1 + if (aliveFrontends.size() > 1) { + fe2 = aliveFrontends[1]["Host"] + ":" + aliveFrontends[1]["QueryPort"] + } + + log.info("fe1: ${fe1}") + log.info("fe2: ${fe2}") + + def dbName = context.config.getDbNameByFile(context.file) + + log.info("connect to fe: ${fe1}") + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = "jdbc:mysql://${fe1}") { + sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')" + + sql "use ${dbName}" + + createTestTable "test_use_plan_cache18" + + sql "sync" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache18" + sql "select * from test_use_plan_cache18" + assertHasCache "select * from test_use_plan_cache18" + } + + log.info("connect to fe: ${fe2}") + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = "jdbc:mysql://${fe2}") { + sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')" + + sql "use ${dbName}" + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache18" + sql "select * from test_use_plan_cache18" + assertHasCache "select * from test_use_plan_cache18" + } }) ).get() } From 002dc70642261c61336115fe5a9447e0187f080d Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 16 Apr 2024 21:06:03 +0800 Subject: [PATCH 20/20] save2 --- .../apache/doris/qe/cache/CacheAnalyzer.java | 22 ++++++--- .../apache/doris/qe/HmsQueryCacheTest.java | 48 +++++++++---------- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index a84c4d68c745e3..161661055a4981 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -428,6 +428,12 @@ private CacheMode innerCheckCacheModeForNereids(long now) { return CacheMode.NoNeed; } + boolean isNewAllViewExpandStmtListStr = allViewExpandStmtListStr == null; + if (isNewAllViewExpandStmtListStr) { + allViewStmtSet.addAll(((LogicalPlanAdapter) parsedStmt).getViewDdlSqls()); + allViewExpandStmtListStr = StringUtils.join(allViewStmtSet, "|"); + } + if (now == 0) { now = nowtime(); } @@ -443,14 +449,16 @@ private CacheMode innerCheckCacheModeForNereids(long now) { .getOriginStatement().originStmt; cache = new SqlCache(this.queryId, originStmt); SqlCache sqlCache = (SqlCache) cache; - NereidsPlanner nereidsPlanner = (NereidsPlanner) planner; - Optional sqlCacheContext = nereidsPlanner - .getCascadesContext() - .getStatementContext() - .getSqlCacheContext(); PUniqueId existsMd5 = null; - if (sqlCacheContext.isPresent()) { - existsMd5 = sqlCacheContext.get().getOrComputeCacheKeyMd5(); + if (planner instanceof NereidsPlanner) { + NereidsPlanner nereidsPlanner = (NereidsPlanner) planner; + Optional sqlCacheContext = nereidsPlanner + .getCascadesContext() + .getStatementContext() + .getSqlCacheContext(); + if (sqlCacheContext.isPresent()) { + existsMd5 = sqlCacheContext.get().getOrComputeCacheKeyMd5(); + } } sqlCache.setCacheInfo(this.latestTable, allViewExpandStmtListStr); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index 7613a2d284b91c..949b28491d6dba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -367,9 +367,9 @@ public void testHitSqlCache() throws Exception { List scanNodes = Arrays.asList(hiveScanNode1); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheMode(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); SqlCache sqlCache = (SqlCache) ca.getCache(); - Assert.assertEquals(sqlCache.getLatestTime(), NOW); + Assert.assertEquals(NOW, sqlCache.getLatestTime()); } @Test @@ -383,11 +383,11 @@ public void testHitSqlCacheAfterPartitionUpdateTimeChanged() throws Exception { CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheMode(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); SqlCache sqlCache1 = (SqlCache) ca.getCache(); // latestTime is equals to the schema update time if not set partition update time - Assert.assertEquals(sqlCache1.getLatestTime(), tbl2.getSchemaUpdateTime()); + Assert.assertEquals(tbl2.getSchemaUpdateTime(), sqlCache1.getLatestTime()); // wait a second and set partition update time try { @@ -401,7 +401,7 @@ public void testHitSqlCacheAfterPartitionUpdateTimeChanged() throws Exception { // check cache mode again ca.checkCacheMode(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); SqlCache sqlCache2 = (SqlCache) ca.getCache(); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); // the latest time will be changed and is equals to the partition update time Assert.assertEquals(later, sqlCache2.getLatestTime()); @@ -415,9 +415,9 @@ public void testHitSqlCacheByNereids() { List scanNodes = Arrays.asList(hiveScanNode1); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheModeForNereids(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); SqlCache sqlCache = (SqlCache) ca.getCache(); - Assert.assertEquals(sqlCache.getLatestTime(), NOW); + Assert.assertEquals(NOW, sqlCache.getLatestTime()); } @Test @@ -431,11 +431,11 @@ public void testHitSqlCacheByNereidsAfterPartitionUpdateTimeChanged() { CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheModeForNereids(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); SqlCache sqlCache1 = (SqlCache) ca.getCache(); // latestTime is equals to the schema update time if not set partition update time - Assert.assertEquals(sqlCache1.getLatestTime(), tbl2.getSchemaUpdateTime()); + Assert.assertEquals(tbl2.getSchemaUpdateTime(), sqlCache1.getLatestTime()); // wait a second and set partition update time try { @@ -449,7 +449,7 @@ public void testHitSqlCacheByNereidsAfterPartitionUpdateTimeChanged() { // check cache mode again ca.checkCacheModeForNereids(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); SqlCache sqlCache2 = (SqlCache) ca.getCache(); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); // the latest time will be changed and is equals to the partition update time Assert.assertEquals(later, sqlCache2.getLatestTime()); @@ -463,9 +463,9 @@ public void testHitSqlCacheWithHiveView() throws Exception { List scanNodes = Arrays.asList(hiveScanNode2); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheMode(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); SqlCache sqlCache = (SqlCache) ca.getCache(); - Assert.assertEquals(sqlCache.getLatestTime(), NOW); + Assert.assertEquals(NOW, sqlCache.getLatestTime()); } @Test @@ -475,9 +475,9 @@ public void testHitSqlCacheWithHiveViewByNereids() { List scanNodes = Arrays.asList(hiveScanNode2); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheModeForNereids(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); SqlCache sqlCache = (SqlCache) ca.getCache(); - Assert.assertEquals(sqlCache.getLatestTime(), NOW); + Assert.assertEquals(NOW, sqlCache.getLatestTime()); } @Test @@ -487,13 +487,13 @@ public void testHitSqlCacheWithNestedHiveView() throws Exception { List scanNodes = Arrays.asList(hiveScanNode3); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheMode(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); Assert.assertEquals(cacheKey, "SELECT `hms_ctl`.`hms_db`.`hms_view2`.`k1` AS `k1` " + "FROM `hms_ctl`.`hms_db`.`hms_view2`" + "|SELECT * FROM hms_db.hms_tbl|SELECT * FROM hms_db.hms_view1"); - Assert.assertEquals(sqlCache.getLatestTime(), NOW); + Assert.assertEquals(NOW, sqlCache.getLatestTime()); } @Test @@ -503,12 +503,12 @@ public void testHitSqlCacheWithNestedHiveViewByNereids() { List scanNodes = Arrays.asList(hiveScanNode3); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheModeForNereids(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + Assert.assertEquals(CacheAnalyzer.CacheMode.Sql, ca.getCacheMode()); SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "select * from hms_ctl.hms_db.hms_view2" - + "|SELECT * FROM hms_db.hms_tbl|SELECT * FROM hms_db.hms_view1"); - Assert.assertEquals(sqlCache.getLatestTime(), NOW); + Assert.assertEquals("select * from hms_ctl.hms_db.hms_view2" + + "|SELECT * FROM hms_db.hms_tbl|SELECT * FROM hms_db.hms_view1", cacheKey); + Assert.assertEquals(NOW, sqlCache.getLatestTime()); } @Test @@ -518,7 +518,7 @@ public void testNotHitSqlCache() throws Exception { List scanNodes = Arrays.asList(hiveScanNode1); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheMode(0); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.None); + Assert.assertEquals(CacheAnalyzer.CacheMode.None, ca.getCacheMode()); } @Test @@ -528,7 +528,7 @@ public void testNotHitSqlCacheByNereids() { List scanNodes = Arrays.asList(hiveScanNode1); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheModeForNereids(0); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.None); + Assert.assertEquals(CacheAnalyzer.CacheMode.None, ca.getCacheMode()); } @Test @@ -540,7 +540,7 @@ public void testNotHitSqlCacheWithFederatedQuery() throws Exception { List scanNodes = Arrays.asList(hiveScanNode1, olapScanNode); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheMode(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.None); + Assert.assertEquals(CacheAnalyzer.CacheMode.None, ca.getCacheMode()); } @Test @@ -552,6 +552,6 @@ public void testNotHitSqlCacheWithFederatedQueryByNereids() { List scanNodes = Arrays.asList(hiveScanNode1, olapScanNode); CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); ca.checkCacheModeForNereids(System.currentTimeMillis() + Config.cache_last_version_interval_second * 1000L * 2); - Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.None); + Assert.assertEquals(CacheAnalyzer.CacheMode.None, ca.getCacheMode()); } }