Skip to content

Commit

Permalink
[Fix](cache) fix query cache returns wrong result after deleting part…
Browse files Browse the repository at this point in the history
…itions.
  • Loading branch information
王翔宇 committed Aug 29, 2023
1 parent d750939 commit 75ebf76
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ private CacheMode innerCheckCacheMode(long now) {
}
if (enablePartitionCache() && ((OlapScanNode) node).getSelectedPartitionNum() > 1
&& selectStmt.hasGroupByClause()) {
LOG.debug("more than one partition scanned when qeury has agg, partition cache cannot use, queryid {}",
LOG.debug("more than one partition scanned when query has agg, partition cache cannot use, queryid {}",
DebugUtil.printId(queryId));
return CacheMode.None;
}
Expand Down Expand Up @@ -583,13 +583,13 @@ private CacheTable getSelectedPartitionLastUpdateTime(OlapScanNode node) {
CacheTable cacheTable = new CacheTable();
OlapTable olapTable = node.getOlapTable();
cacheTable.olapTable = olapTable;
cacheTable.partitionNum = node.getSelectedPartitionIds().size();
for (Long partitionId : node.getSelectedPartitionIds()) {
Partition partition = olapTable.getPartition(partitionId);
if (partition.getVisibleVersionTime() >= cacheTable.latestTime) {
cacheTable.latestPartitionId = partition.getId();
cacheTable.latestTime = partition.getVisibleVersionTime();
cacheTable.latestVersion = partition.getVisibleVersion();
cacheTable.partitionNum = node.getSelectedPartitionNum();
}
}
return cacheTable;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ public String getSqlWithViewStmt() {
return cacheKey;
}

public long getSumOfPartitionNum() {
return latestTable.sumOfPartitionNum;
}

public InternalService.PFetchCacheResult getCacheData(Status status) {
InternalService.PFetchCacheRequest request = InternalService.PFetchCacheRequest.newBuilder()
.setSqlKey(CacheProxy.getMd5(getSqlWithViewStmt()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1104,6 +1104,7 @@ public void testSqlCacheKey() {
Assert.assertEquals(cacheKey, "SELECT <slot 2> `eventdate` AS `eventdate`, <slot 3> count(`userid`) AS "
+ "`count(``userid``)` FROM `testCluster:testDb`.`appevent` WHERE `eventdate` >= '2020-01-12' AND "
+ "`eventdate` <= '2020-01-14' GROUP BY `eventdate`|");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}

@Test
Expand All @@ -1123,6 +1124,7 @@ public void testSqlCacheKeyWithChineseChar() {
String cacheKey = sqlCache.getSqlWithViewStmt();
Types.PUniqueId sqlKey2 = CacheProxy.getMd5(cacheKey.replace("北京", "上海"));
Assert.assertNotEquals(CacheProxy.getMd5(sqlCache.getSqlWithViewStmt()), sqlKey2);
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}

@Test
Expand All @@ -1141,6 +1143,7 @@ public void testSqlCacheKeyWithView() {
Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view1`.`eventdate` AS `eventdate`, `testCluster:testDb`.`view1`."
+ "`count(`userid`)` AS `count(``userid``)` FROM `testCluster:testDb`.`view1`|select eventdate, COUNT(userid) "
+ "FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}

@Test
Expand All @@ -1159,6 +1162,7 @@ public void testSqlCacheKeyWithViewForNereids() {
Assert.assertEquals(cacheKey, "SELECT * from testDb.view1"
+ "|select eventdate, COUNT(userid) FROM appevent "
+ "WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}

@Test
Expand All @@ -1185,6 +1189,7 @@ public void testSqlCacheKeyWithSubSelectView() {
+ "`userid` FROM (SELECT `view2`.`eventdate` AS `eventdate`, `view2`.`userid` AS `userid` FROM "
+ "`testCluster:testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12' AND `view2`.`eventdate` "
+ "<= '2020-01-14') origin|select eventdate, userid FROM appevent");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}


Expand Down Expand Up @@ -1214,6 +1219,7 @@ public void testSqlCacheKeyWithSubSelectViewForNereids() {
+ " from testDb.view2 view2 \n"
+ " where view2.eventdate >=\"2020-01-12\" and view2.eventdate <= \"2020-01-14\"\n"
+ ") origin" + "|select eventdate, userid FROM appevent");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}

@Test
Expand Down Expand Up @@ -1292,6 +1298,7 @@ public void testSqlCacheKeyWithNestedView() {
+ "`testCluster:testDb`.`view4`.`count(`userid`)` AS `count(``userid``)` FROM `testCluster:testDb`.`view4`|select "
+ "eventdate, COUNT(userid) FROM view2 WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-14\" GROUP BY eventdate|select eventdate, userid FROM appevent");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}

@Test
Expand All @@ -1311,6 +1318,7 @@ public void testSqlCacheKeyWithNestedViewForNereids() {
+ "|select eventdate, COUNT(userid) FROM view2 "
+ "WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"
+ "|select eventdate, userid FROM appevent");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}

@Test
Expand All @@ -1328,10 +1336,12 @@ public void testCacheLocalViewMultiOperand() {
);
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L);
List<ScanNode> scanNodes = Lists.newArrayList(createProfileScanNode(selectedPartitionIds));
ScanNode scanNode = createProfileScanNode(selectedPartitionIds);
List<ScanNode> scanNodes = Lists.newArrayList(scanNode, scanNode, scanNode);
CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
ca.checkCacheMode(0);
Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
Assert.assertEquals(selectedPartitionIds.size() * 3, ((SqlCache) ca.getCache()).getSumOfPartitionNum());
}

@Test
Expand Down

0 comments on commit 75ebf76

Please sign in to comment.