Skip to content

[regression](filecache) fix regression failures part2 #53783

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gavinchou merged 4 commits into from
Jul 24, 2025
Merged

[regression](filecache) fix regression failures part2 #53783

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a169c2
fix 4
freemandealer Jul 15, 2025
f3f81f0
fix 5
freemandealer Jul 17, 2025
d93fbc7
fix 6
freemandealer Jul 23, 2025
3cd544b
minor fix
freemandealer Jul 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ suite("test_reset_capacity") {
continue
}
def i = line.indexOf(' ')
ttl_cache_size = line.substring(i).toLong()
def ttl_cache_size = line.substring(i).toLong()
logger.info("current ttl_cache_size " + ttl_cache_size);
assertTrue(ttl_cache_size <= 1073741824)
flag1 = true
Expand Down
5 changes: 2 additions & 3 deletions ...sion-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,11 +163,10 @@ suite("test_warm_up_cluster") {
for (; i < retryTime; i++) {
sleep(1000)
def statuses = getJobState(jobId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand Down
5 changes: 2 additions & 3 deletions ...est/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,10 @@ suite("test_warm_up_cluster_batch") {
for (; i < retryTime; i++) {
sleep(1000)
def statuses = getJobState(jobId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand Down
5 changes: 2 additions & 3 deletions ...t/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,10 @@ suite("test_warm_up_cluster_bigsize") {
for (; i < retryTime; i++) {
sleep(1000)
def statuses = getJobState(jobId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand Down
5 changes: 2 additions & 3 deletions ...est/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,10 @@ suite("test_warm_up_cluster_empty") {
for (; i < retryTime; i++) {
sleep(1000)
def statuses = getJobState(jobId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand Down
5 changes: 2 additions & 3 deletions ...est/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,11 +158,10 @@ suite("test_warm_up_compute_group") {
for (; i < retryTime; i++) {
sleep(1000)
def statuses = getJobState(jobId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand Down
7 changes: 3 additions & 4 deletions ...sion-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,12 +126,11 @@ suite("test_warm_up_partition") {
int i = 0
for (; i < retryTime; i++) {
sleep(1000)
def statuses = getJobState(jodId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
def statuses = getJobState(jobId[0][0])
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand Down
12 changes: 5 additions & 7 deletions ...tes/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,10 @@ suite("test_warm_up_same_table_multi_times") {
for (; j < retryTime; j++) {
sleep(1000)
def statuses = getJobState(jobId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand Down Expand Up @@ -197,12 +196,11 @@ suite("test_warm_up_same_table_multi_times") {
j = 0
for (; j < retryTime; j++) {
sleep(1000)
def status = getJobState(jobId[0][0])
logger.info(status)
if (status.equals("CANCELLED")) {
def statuses = getJobState(jobId[0][0])
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (status.equals("FINISHED")) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand Down
11 changes: 5 additions & 6 deletions regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@ suite("test_warm_up_table") {
}
def getTablesFromShowCommand = { jobId ->
def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """
logger.info(jobStateResult)
return jobStateResult[0][9]
return jobStateResult[0]
}

List<String> ipList = new ArrayList<>();
Expand Down Expand Up @@ -153,11 +152,10 @@ suite("test_warm_up_table") {
for (; j < retryTime; j++) {
sleep(1000)
def statuses = getJobState(jobId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand All @@ -166,7 +164,8 @@ suite("test_warm_up_table") {
assertTrue(false);
}
def tablesString = getTablesFromShowCommand(jobId[0][0])
assertTrue(tablesString.contains("customer"), tablesString)

assertTrue(tablesString.any { it != null && it.contains("customer") })
sleep(30000)
long ttl_cache_size = 0
getMetricsMethod.call(ipList[0], brpcPortList[0]) {
Expand Down
12 changes: 5 additions & 7 deletions regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@ suite("test_warm_up_tables") {
}
def getTablesFromShowCommand = { jobId ->
def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """
logger.info(jobStateResult)
return jobStateResult[0][9]
return jobStateResult[0]
}

List<String> ipList = new ArrayList<>();
Expand Down Expand Up @@ -157,11 +156,10 @@ suite("test_warm_up_tables") {
for (; i < retryTime; i++) {
sleep(1000)
def statuses = getJobState(jobId[0][0])
logger.info(statuses)
if (statuses.any { it.equals("CANCELLED") }) {
if (statuses.any { it != null && it.equals("CANCELLED") }) {
assertTrue(false);
}
if (statuses.any { it.equals("FINISHED") }) {
if (statuses.any { it != null && it.equals("FINISHED") }) {
break;
}
}
Expand All @@ -176,8 +174,8 @@ suite("test_warm_up_tables") {
waitJobDone(jobId_);

def tablesString = getTablesFromShowCommand(jobId_[0][0])
assertTrue(tablesString.contains("customer.p3"), tablesString)
assertTrue(tablesString.contains("supplier"), tablesString)
assertTrue(tablesString.any { it != null && it.contains("customer") })
assertTrue(tablesString.any { it != null && it.contains("supplier") })

sleep(30000)
long ttl_cache_size = 0
Expand Down
44 changes: 13 additions & 31 deletions regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ suite("alter_ttl_1") {
]

setBeConfigTemporary(custoBeConfig) {
sql "set global enable_auto_analyze = false"
def clusters = sql " SHOW CLUSTERS; "
assertTrue(!clusters.isEmpty())
def validCluster = clusters[0][0]
Expand Down Expand Up @@ -112,12 +113,14 @@ suite("alter_ttl_1") {
sql """ select count(*) from customer_ttl """
sleep(30000)
long ttl_cache_size = 0
long normal_cache_size = 0
getMetricsMethod.call() {
respCode, body ->
assertEquals("${respCode}".toString(), "200")
String out = "${body}".toString()
def strs = out.split('\n')
Boolean flag1 = false;
Boolean flag2 = false;
for (String line in strs) {
if (flag1) break;
if (line.contains("ttl_cache_size")) {
Expand All @@ -128,64 +131,43 @@ suite("alter_ttl_1") {
ttl_cache_size = line.substring(i).toLong()
flag1 = true
}
}
assertTrue(flag1)
}
sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="140") """
sleep(80000)
// after 110s, the first load has translate to normal
getMetricsMethod.call() {
respCode, body ->
assertEquals("${respCode}".toString(), "200")
String out = "${body}".toString()
def strs = out.split('\n')
Boolean flag1 = false;
for (String line in strs) {
if (flag1) break;
if (line.contains("ttl_cache_size")) {
if (line.startsWith("#")) {
continue
}
def i = line.indexOf(' ')
assertEquals(line.substring(i).toLong(), 0)

}

if (line.contains("normal_queue_cache_size")) {
if (line.startsWith("#")) {
continue
}
def i = line.indexOf(' ')
assertEquals(line.substring(i).toLong(), ttl_cache_size)
flag1 = true
normal_cache_size = line.substring(i).toLong()
flag2 = true
}
}
assertTrue(flag1)
assertTrue(flag1 && flag2)
}
// wait for ttl timeout
sleep(50000)
sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="100") """
sleep(80000)
// after 110s, the first load has translate to normal
getMetricsMethod.call() {
respCode, body ->
assertEquals("${respCode}".toString(), "200")
String out = "${body}".toString()
def strs = out.split('\n')
Boolean flag1 = false;
Boolean flag2 = false;
for (String line in strs) {
if (flag1 && flag2) break;
if (flag1) break;
if (line.contains("ttl_cache_size")) {
if (line.startsWith("#")) {
continue
}
def i = line.indexOf(' ')
assertEquals(line.substring(i).toLong(), 0)

}

if (line.contains("normal_queue_cache_size")) {
if (line.startsWith("#")) {
continue
}
def i = line.indexOf(' ')
assertEquals(line.substring(i).toLong(), ttl_cache_size)
assertEquals(line.substring(i).toLong(), ttl_cache_size + normal_cache_size)
flag1 = true
}
}
Expand Down
8 changes: 7 additions & 1 deletion regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ suite("alter_ttl_4") {
}
sleep(60000)
// one customer table would take about 1.3GB, the total cache size is 20GB
// the following would take 20.8G all
// the following would take 20G all
// evict customer_ttl
load_customer_once("customer")
load_customer_once("customer")
Expand All @@ -175,6 +175,12 @@ suite("alter_ttl_4") {
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")

// some datas in s3 and will download them
sql """ select C_CUSTKEY from customer_ttl order by C_CUSTKEY limit 1"""
Expand Down
23 changes: 0 additions & 23 deletions regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,29 +144,6 @@ def clearFileCache = { check_func ->
DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 32
PROPERTIES("file_cache_ttl_seconds"="120","disable_auto_compaction" = "true") as select * from customer_ttl"""

sleep(30000) // 30s
getMetricsMethod.call() {
respCode, body ->
assertEquals("${respCode}".toString(), "200")
String out = "${body}".toString()
def strs = out.split('\n')
Boolean flag1 = false;
Boolean flag2 = false;
for (String line in strs) {
if (flag1 && flag2) break;
if (line.contains("ttl_cache_size")) {
if (line.startsWith("#")) {
continue
}
def i = line.indexOf(' ')
long cur_ttl_cache_size = line.substring(i).toLong()
assertTrue(Math.abs(2* ttl_cache_size - cur_ttl_cache_size) < 10000)
flag1 = true
}
}
assertTrue(flag1)
}

sleep(150000)
getMetricsMethod.call() {
respCode, body ->
Expand Down
9 changes: 8 additions & 1 deletion regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,14 @@ suite("test_ttl_preempt") {
}

// one customer table would take about 1.3GB, the total cache size is 20GB
// the following would take 19.5G all
// the following would take 20G all
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
load_customer_once("customer")
Expand Down
Loading