Merge #33011

33011: roachtest: don't fail tests based on slow health checker r=petermattis a=tbg

We know there can be a backlog of Raft snapshots at the beginning of the
test. This isn't ideal, but we know about it and have #32046 tracking it.

Closes #32859.

Release note: None

Co-authored-by: Tobias Schottdorf <tobias.schottdorf@gmail.com>

pkg/cmd/roachtest/restore.go

@@ -110,7 +110,7 @@ func (hc *HealthChecker) Runner(ctx context.Context) (err error) {
 		}
 		// TODO(tschottdorf): remove replicate queue failures when the cluster first starts.
 		// Ditto queue.raftsnapshot.process.failure.
-		rows, err := db.QueryContext(ctx, `SELECT * FROM crdb_internal.gossip_alerts WHERE description != 'queue.replicate.process.failure' AND description != 'ranges.underreplicated' AND description != 'queue.raftsnapshot.process.failure' ORDER BY node_id ASC, store_id ASC`)
+		rows, err := db.QueryContext(ctx, `SELECT * FROM crdb_internal.gossip_alerts ORDER BY node_id ASC, store_id ASC`)
 		_ = db.Close()
 		if err != nil {
 			return err
@@ -134,7 +134,10 @@ func (hc *HealthChecker) Runner(ctx context.Context) (err error) {
 		}
 
 		if elapsed := timeutil.Since(tBegin); elapsed > 10*time.Second {
-			return errors.Errorf("health check against node %d took %s", nodeIdx, elapsed)
+			err := errors.Errorf("health check against node %d took %s", nodeIdx, elapsed)
+			logger.Printf(err.Error() + "\n")
+			// TODO(tschottdorf): see method comment.
+			// return err
 		}
 	}
 }