From 2322365cd15f300ef7cd3b97e234d326c28e7941 Mon Sep 17 00:00:00 2001 From: Ricky Stewart Date: Fri, 10 Nov 2023 21:58:38 +0000 Subject: [PATCH] github-pull-request-make: pass timeout to `stress` directly We're seeing a lot of timeouts here. I think it's because the test timeouts are being passed incorrectly. If you look at a test run that timed out, the command looks like the following: ``` bazci test ... --test_arg=-test.timeout --test_arg=7m30s --test_timeout=660 ... -maxfails 1 -maxtime 10m0s -p 16 ``` When you look at this, the problem may be obvious: * We tell `stress` to run the test for up to 10 minutes; but * We tell `bazel` the timeout for the test should be 7m30s. In this way, it seems it's obvious tests are timing out. The shorter timeout (7m30s) is meant to represent the timeout for the individual test process under-the-hood. To me that means that we should instead pass the timeout to `stress` as `timeout`, which is what this PR does. Epic: none Release note: None --- pkg/cmd/github-pull-request-make/main.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/cmd/github-pull-request-make/main.go b/pkg/cmd/github-pull-request-make/main.go index 7d92067619b7..bca5df68eda7 100644 --- a/pkg/cmd/github-pull-request-make/main.go +++ b/pkg/cmd/github-pull-request-make/main.go @@ -279,9 +279,9 @@ func main() { target, ok := os.LookupEnv(targetEnv) var duration time.Duration if ok && target == "stressrace" { - duration = (40 * time.Minute) / time.Duration(len(pkgs)) - } else { duration = (30 * time.Minute) / time.Duration(len(pkgs)) + } else { + duration = (20 * time.Minute) / time.Duration(len(pkgs)) } minDuration := (2 * time.Minute) * time.Duration(len(pkg.tests)) if duration < minDuration { @@ -354,12 +354,11 @@ func main() { } args = append(args, fmt.Sprintf("--test_filter=%s", strings.Join(filters, "|"))) args = append(args, "--test_env=COCKROACH_NIGHTLY_STRESS=true") - args = append(args, "--test_arg=-test.timeout", fmt.Sprintf("--test_arg=%s", timeout)) // Give the entire test 1 more minute than the duration to wrap up. args = append(args, fmt.Sprintf("--test_timeout=%d", int((duration+1*time.Minute).Seconds()))) args = append(args, "--test_output", "streamed") - args = append(args, "--run_under", fmt.Sprintf("%s -bazel -shardable-artifacts 'XML_OUTPUT_FILE=%s merge-test-xmls' -stderr -maxfails 1 -maxtime %s -p %d", bazelStressTarget, bazciPath, duration, parallelism)) + args = append(args, "--run_under", fmt.Sprintf("%s -bazel -shardable-artifacts 'XML_OUTPUT_FILE=%s merge-test-xmls' -stderr -maxfails 1 -maxtime %s -p %d -timeout %s", bazelStressTarget, bazciPath, duration, parallelism, timeout)) cmd := exec.Command("bazci", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr