roachtest: add a retry loop around network activity

This is a first attempt to try to address #31127 and #31298 by putting a retry
loop around anything that does any downloading. We've seen a number of flakes
because of iffy networking and this should hopefully solve the problem.

If this doesn't fix the issues, the next step it to add some alternative
sources to sources.list.d.

Release note: None

pkg/cmd/roachtest/cluster.go

@@ -939,11 +939,14 @@ func (c *cluster) Put(ctx context.Context, src, dest string, opts ...option) {
 	}
 }
 
-// GitClone clones a git repo from src into dest and checks out
-// origin's version of the given branch. The src, dest, and branch
-// arguments must not contain shell special characters.
-func (c *cluster) GitClone(ctx context.Context, src, dest, branch string, node nodeListOption) {
-	c.Run(ctx, node, "bash", "-e", "-c", fmt.Sprintf(`'
+// GitCloneE clones a git repo from src into dest and checks out origin's
+// version of the given branch. The src, dest, and branch arguments must not
+// contain shell special characters. GitCloneE unlike GitClone returns an
+// error.
+func (c *cluster) GitCloneE(
+	ctx context.Context, src, dest, branch string, node nodeListOption,
+) error {
+	return c.RunE(ctx, node, "bash", "-e", "-c", fmt.Sprintf(`'
 if ! test -d %s; then
   git clone -b %s --depth 1 %s %s
 else
@@ -957,6 +960,15 @@ fi
 		branch))
 }
 
+// GitClone clones a git repo from src into dest and checks out origin's
+// version of the given branch. The src, dest, and branch arguments must not
+// contain shell special characters.
+func (c *cluster) GitClone(ctx context.Context, src, dest, branch string, node nodeListOption) {
+	if err := c.GitCloneE(ctx, src, dest, branch, node); err != nil {
+		c.t.Fatal(err)
+	}
+}
+
 // startArgs specifies extra arguments that are passed to `roachprod` during `c.Start`.
 func startArgs(extraArgs ...string) option {
 	return roachprodArgOption(extraArgs)

pkg/cmd/roachtest/hibernate.go

@@ -21,6 +21,9 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/util/retry"
 )
 
 // This test runs hibernate-core's full test suite against an single cockroach
@@ -41,28 +44,89 @@ func registerHibernate(r *registry) {
 		c.Start(ctx, t, c.All())
 
 		t.Status("cloning hibernate and installing prerequisites")
-		c.Run(ctx, node, `rm -rf /mnt/data1/hibernate`)
-		c.GitClone(ctx,
-			"https://github.com/hibernate/hibernate-orm.git", "/mnt/data1/hibernate", "5.3.6", node,
-		)
-		c.Run(ctx, node, `sudo apt-get -q update`)
-		c.Run(ctx, node, `sudo apt-get -qy install default-jre openjdk-8-jdk-headless gradle`)
-
-		// In order to get Hibernate's test suite to connect to cockroach, we have
-		// to create a dbBundle as it not possible to specify the individual
-		// properties. So here we just steamroll the file with our own config.
-		c.Run(ctx, node, `echo "ext {
-					db = project.hasProperty('db') ? project.getProperty('db') : 'h2'
-					dbBundle = [
-									cockroach : [
-													'db.dialect' : 'org.hibernate.dialect.PostgreSQL95Dialect',
-													'jdbc.driver': 'org.postgresql.Driver',
-													'jdbc.user'  : 'root',
-													'jdbc.pass'  : '',
-													'jdbc.url'   : 'jdbc:postgresql://localhost:26257/defaultdb?sslmode=disable'
-									],
-					]
-		}" > /mnt/data1/hibernate/gradle/databases.gradle`)
+		opts := retry.Options{
+			InitialBackoff: 10 * time.Second,
+			Multiplier:     2,
+			MaxBackoff:     5 * time.Minute,
+		}
+		for attempt, r := 0, retry.StartWithCtx(ctx, opts); r.Next(); {
+			if ctx.Err() != nil {
+				return
+			}
+			if c.t.Failed() {
+				return
+			}
+			attempt++
+
+			c.l.Printf("attempt %d - update dependencies", attempt)
+			if err := c.RunE(ctx, node, `sudo apt-get -q update`); err != nil {
+				continue
+			}
+			if err := c.RunE(
+				ctx, node, `sudo apt-get -qy install default-jre openjdk-8-jdk-headless gradle`,
+			); err != nil {
+				continue
+			}
+
+			c.l.Printf("attempt %d - cloning hibernate", attempt)
+			if err := c.RunE(ctx, node, `rm -rf /mnt/data1/hibernate`); err != nil {
+				continue
+			}
+			if err := c.GitCloneE(
+				ctx,
+				"https://github.com/hibernate/hibernate-orm.git",
+				"/mnt/data1/hibernate",
+				"5.3.6",
+				node,
+			); err != nil {
+				continue
+			}
+
+			// In order to get Hibernate's test suite to connect to cockroach, we have
+			// to create a dbBundle as it not possible to specify the individual
+			// properties. So here we just steamroll the file with our own config.
+			if err := c.RunE(ctx, node, `
+echo "ext {
+  db = project.hasProperty('db') ? project.getProperty('db') : 'h2'
+    dbBundle = [
+     cockroach : [
+       'db.dialect' : 'org.hibernate.dialect.PostgreSQL95Dialect',
+       'jdbc.driver': 'org.postgresql.Driver',
+       'jdbc.user'  : 'root',
+       'jdbc.pass'  : '',
+       'jdbc.url'   : 'jdbc:postgresql://localhost:26257/defaultdb?sslmode=disable'
+     ],
+    ]
+}" > /mnt/data1/hibernate/gradle/databases.gradle`,
+			); err != nil {
+				continue
+			}
+
+			break
+		}
+
+		t.Status("building hibernate (without tests)")
+		for attempt, r := 0, retry.StartWithCtx(ctx, opts); r.Next(); {
+			if ctx.Err() != nil {
+				return
+			}
+			if c.t.Failed() {
+				return
+			}
+			attempt++
+			c.l.Printf("attempt %d - building hibernate", attempt)
+			// Build hibernate and run a single test, this step involves some
+			// downloading, so it needs a retry loop as well. Just building was not
+			// enough as the test libraries are not downloaded unless at least a
+			// single test is invoked.
+			if err := c.RunE(
+				ctx, node, `cd /mnt/data1/hibernate/hibernate-core/ && ./../gradlew test -Pdb=cockroach `+
+					`--tests org.hibernate.jdbc.util.BasicFormatterTest.*`,
+			); err != nil {
+				continue
+			}
+			break
+		}
 
 		t.Status("running hibernate test suite, will take at least 3 hours")
 		// When testing, it is helpful to run only a subset of the tests. To do so