test(testutil): try retrying for 'panic: pebbledb: closed'

CHANGELOG.md

@@ -75,6 +75,7 @@ needed to include double quotes around the hexadecimal string.
 - [#2156](https://github.com/NibiruChain/nibiru/pull/2156) - test(evm-e2e): add E2E test using the Nibiru Oracle's ChainLink impl
 - [#2157](https://github.com/NibiruChain/nibiru/pull/2157) - fix(evm): Fix unit inconsistency related to AuthInfo.Fee and txData.Fee using effective fee
 - [#2160](https://github.com/NibiruChain/nibiru/pull/2160) - fix(evm-precompile): use bank.MsgServer Send in precompile IFunToken.bankMsgSend
+- [#2162](https://github.com/NibiruChain/nibiru/pull/2162) - test(testutil): try retrying for 'panic: pebbledb: closed'
 
 #### Nibiru EVM | Before Audit 2 - 2024-12-06
 

app/wasmext/wasm_cli_test/cli_test.go

@@ -140,5 +140,7 @@ func (s *TestSuite) requiredDeployedContractsLen(total int) {
 }
 
 func TestIntegrationTestSuite(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }

eth/rpc/rpcapi/eth_api_test.go

@@ -58,7 +58,10 @@ type NodeSuite struct {
 
 func TestSuite_RunAll(t *testing.T) {
 	suite.Run(t, new(Suite))
-	suite.Run(t, new(NodeSuite))
+
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(NodeSuite))
+	}, 2)
 }
 
 // SetupSuite runs before every test in the suite. Implements the

x/common/testutil/cases.go

@@ -1,6 +1,7 @@
 package testutil
 
 import (
+	"strings"
 	"testing"
 )
 
@@ -34,3 +35,40 @@
 	}
 	suiteT.Log("setting up integration test suite")
 }
+
+// RetrySuiteRunIfDbClosed runs a test suite with retries, recovering from a
+// specific panic message, "pebbledb: closed" that often surfaces in CI when tests
+// involve "Nibiru/x/common/testutil/testnetwork".
+// For full context, see https://github.com/NibiruChain/nibiru/issues/1918.
+func RetrySuiteRunIfDbClosed(t *testing.T, runTest func(), maxRetries int) {
+	panicMessage := "pebbledb: closed"
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		panicked := false
+
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					if errMsg, ok := r.(string); ok && strings.Contains(errMsg, panicMessage) {
+						t.Logf("Recovered from panic on attempt %d: %v", attempt, r)
+						panicked = true
+					} else {
+						panic(r) // Re-panic if it's not the specific error
+					}
+				}
+			}()
+
+			// Run the test suite
+			runTest()
+			// suite.Run(t, suiteInstance)
+		}()
+
+		if !panicked {
+			t.Logf("Test suite succeeded on attempt %d", attempt)
+			return
+		}
+
+		t.Logf("Retrying test suite: attempt %d", attempt+1)
+	}
+
+	t.Fatalf("Test suite failed after %d attempts due to '%s'", maxRetries, panicMessage)
+}

x/common/testutil/testnetwork/network_test.go

@@ -22,7 +22,9 @@ import (
 )
 
 func TestIntegrationTestSuite_RunAll(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
 
 // Assert network cleanup

x/oracle/keeper/app_test.go

@@ -177,7 +177,9 @@ func (s *TestSuite) currentPrices() map[asset.Pair]sdk.Dec {
 }
 
 func TestIntegrationTestSuite(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
 
 func (s *TestSuite) TearDownSuite() {

x/sudo/cli/cli_test.go

@@ -96,7 +96,9 @@ type Account struct {
 }
 
 func TestSuite_IntegrationSuite_RunAll(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
 
 // ———————————————————————————————————————————————————————————————————

x/tokenfactory/cli/cli_test.go

@@ -31,7 +31,9 @@ type TestSuite struct {
 }
 
 func TestIntegrationTestSuite(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
 
 // TestTokenFactory: Runs the test suite with a deterministic order.