bench: ignore cluster startup and shutdown

This commit adjusts our benchmark harness to ignore the overhead of
starting up and shutting down the test cluster. This allows us to avoid
spurious "regressions" like that we've seen due to cockroachdb#117116.

Epic: None

Release note: None

pkg/bench/foreachdb.go

@@ -42,6 +42,14 @@ var runSepProcessTenant = flag.Bool("run-sep-process-tenant", false, "run separa
 // BenchmarkFn is a function that runs a benchmark using the given SQLRunner.
 type BenchmarkFn func(b *testing.B, db *sqlutils.SQLRunner)
 
+// timerUtil is a helper method that should be called right before the
+// invocation of BenchmarkFn and the returned function should be deferred.
+func timerUtil(b *testing.B) func() {
+	b.ResetTimer()
+	b.StartTimer()
+	return b.StopTimer
+}
+
 func benchmarkCockroach(b *testing.B, f BenchmarkFn) {
 	s, db, _ := serverutils.StartServer(
 		b, base.TestServerArgs{
@@ -54,6 +62,7 @@ func benchmarkCockroach(b *testing.B, f BenchmarkFn) {
 		b.Fatal(err)
 	}
 
+	defer timerUtil(b)()
 	f(b, sqlutils.MakeSQLRunner(db))
 }
 
@@ -103,6 +112,7 @@ func benchmarkSharedProcessTenantCockroach(b *testing.B, f BenchmarkFn) {
 	_, err = tenantDB.Exec(`CREATE DATABASE bench`)
 	require.NoError(b, err)
 
+	defer timerUtil(b)()
 	f(b, sqlutils.MakeSQLRunner(tenantDB))
 }
 
@@ -133,6 +143,7 @@ func benchmarkSepProcessTenantCockroach(b *testing.B, f BenchmarkFn) {
 	_, err = tenantDB.Exec(`CREATE DATABASE bench`)
 	require.NoError(b, err)
 
+	defer timerUtil(b)()
 	f(b, sqlutils.MakeSQLRunner(tenantDB))
 }
 
@@ -150,6 +161,7 @@ func benchmarkMultinodeCockroach(b *testing.B, f BenchmarkFn) {
 	}
 	defer tc.Stopper().Stop(context.TODO())
 
+	defer timerUtil(b)()
 	f(b, sqlutils.MakeRoundRobinSQLRunner(tc.Conns[0], tc.Conns[1], tc.Conns[2]))
 }
 
@@ -198,6 +210,7 @@ func benchmarkPostgres(b *testing.B, f BenchmarkFn) {
 	r := sqlutils.MakeSQLRunner(db)
 	r.Exec(b, `CREATE SCHEMA IF NOT EXISTS bench`)
 
+	defer timerUtil(b)()
 	f(b, r)
 }
 
@@ -218,6 +231,7 @@ func benchmarkMySQL(b *testing.B, f BenchmarkFn) {
 	r := sqlutils.MakeSQLRunner(db)
 	r.Exec(b, `CREATE DATABASE IF NOT EXISTS bench`)
 
+	defer timerUtil(b)()
 	f(b, r)
 }