From a0115e640da9e4b49472280753f90412538f7c88 Mon Sep 17 00:00:00 2001 From: Bram Gruneir Date: Mon, 15 Oct 2018 15:52:33 -0400 Subject: [PATCH] roachtest: add a retry loop around network activity This is a first attempt to try to address #31127 and #31298 by putting a retry loop around anything that does any downloading. We've seen a number of flakes because of iffy networking and this should hopefully solve the problem. If this doesn't fix the issues, the next step it to add some alternative sources to sources.list.d. Release note: None --- pkg/cmd/roachtest/cluster.go | 22 +++++-- pkg/cmd/roachtest/hibernate.go | 108 ++++++++++++++++++++++++++------- 2 files changed, 103 insertions(+), 27 deletions(-) diff --git a/pkg/cmd/roachtest/cluster.go b/pkg/cmd/roachtest/cluster.go index 99bd4d1316e6..078ab1c8ebf0 100644 --- a/pkg/cmd/roachtest/cluster.go +++ b/pkg/cmd/roachtest/cluster.go @@ -939,11 +939,14 @@ func (c *cluster) Put(ctx context.Context, src, dest string, opts ...option) { } } -// GitClone clones a git repo from src into dest and checks out -// origin's version of the given branch. The src, dest, and branch -// arguments must not contain shell special characters. -func (c *cluster) GitClone(ctx context.Context, src, dest, branch string, node nodeListOption) { - c.Run(ctx, node, "bash", "-e", "-c", fmt.Sprintf(`' +// GitCloneE clones a git repo from src into dest and checks out origin's +// version of the given branch. The src, dest, and branch arguments must not +// contain shell special characters. GitCloneE unlike GitClone returns an +// error. +func (c *cluster) GitCloneE( + ctx context.Context, src, dest, branch string, node nodeListOption, +) error { + return c.RunE(ctx, node, "bash", "-e", "-c", fmt.Sprintf(`' if ! test -d %s; then git clone -b %s --depth 1 %s %s else @@ -957,6 +960,15 @@ fi branch)) } +// GitClone clones a git repo from src into dest and checks out origin's +// version of the given branch. The src, dest, and branch arguments must not +// contain shell special characters. +func (c *cluster) GitClone(ctx context.Context, src, dest, branch string, node nodeListOption) { + if err := c.GitCloneE(ctx, src, dest, branch, node); err != nil { + c.t.Fatal(err) + } +} + // startArgs specifies extra arguments that are passed to `roachprod` during `c.Start`. func startArgs(extraArgs ...string) option { return roachprodArgOption(extraArgs) diff --git a/pkg/cmd/roachtest/hibernate.go b/pkg/cmd/roachtest/hibernate.go index a887bde17e32..7268e8bfd372 100644 --- a/pkg/cmd/roachtest/hibernate.go +++ b/pkg/cmd/roachtest/hibernate.go @@ -21,6 +21,9 @@ import ( "fmt" "sort" "strings" + "time" + + "github.com/cockroachdb/cockroach/pkg/util/retry" ) // This test runs hibernate-core's full test suite against an single cockroach @@ -41,28 +44,89 @@ func registerHibernate(r *registry) { c.Start(ctx, t, c.All()) t.Status("cloning hibernate and installing prerequisites") - c.Run(ctx, node, `rm -rf /mnt/data1/hibernate`) - c.GitClone(ctx, - "https://github.com/hibernate/hibernate-orm.git", "/mnt/data1/hibernate", "5.3.6", node, - ) - c.Run(ctx, node, `sudo apt-get -q update`) - c.Run(ctx, node, `sudo apt-get -qy install default-jre openjdk-8-jdk-headless gradle`) - - // In order to get Hibernate's test suite to connect to cockroach, we have - // to create a dbBundle as it not possible to specify the individual - // properties. So here we just steamroll the file with our own config. - c.Run(ctx, node, `echo "ext { - db = project.hasProperty('db') ? project.getProperty('db') : 'h2' - dbBundle = [ - cockroach : [ - 'db.dialect' : 'org.hibernate.dialect.PostgreSQL95Dialect', - 'jdbc.driver': 'org.postgresql.Driver', - 'jdbc.user' : 'root', - 'jdbc.pass' : '', - 'jdbc.url' : 'jdbc:postgresql://localhost:26257/defaultdb?sslmode=disable' - ], - ] - }" > /mnt/data1/hibernate/gradle/databases.gradle`) + opts := retry.Options{ + InitialBackoff: 10 * time.Second, + Multiplier: 2, + MaxBackoff: 5 * time.Minute, + } + for attempt, r := 0, retry.StartWithCtx(ctx, opts); r.Next(); { + if ctx.Err() != nil { + return + } + if c.t.Failed() { + return + } + attempt++ + + c.l.Printf("attempt %d - update dependencies", attempt) + if err := c.RunE(ctx, node, `sudo apt-get -q update`); err != nil { + continue + } + if err := c.RunE( + ctx, node, `sudo apt-get -qy install default-jre openjdk-8-jdk-headless gradle`, + ); err != nil { + continue + } + + c.l.Printf("attempt %d - cloning hibernate", attempt) + if err := c.RunE(ctx, node, `rm -rf /mnt/data1/hibernate`); err != nil { + continue + } + if err := c.GitCloneE( + ctx, + "https://github.com/hibernate/hibernate-orm.git", + "/mnt/data1/hibernate", + "5.3.6", + node, + ); err != nil { + continue + } + + // In order to get Hibernate's test suite to connect to cockroach, we have + // to create a dbBundle as it not possible to specify the individual + // properties. So here we just steamroll the file with our own config. + if err := c.RunE(ctx, node, ` +echo "ext { + db = project.hasProperty('db') ? project.getProperty('db') : 'h2' + dbBundle = [ + cockroach : [ + 'db.dialect' : 'org.hibernate.dialect.PostgreSQL95Dialect', + 'jdbc.driver': 'org.postgresql.Driver', + 'jdbc.user' : 'root', + 'jdbc.pass' : '', + 'jdbc.url' : 'jdbc:postgresql://localhost:26257/defaultdb?sslmode=disable' + ], + ] +}" > /mnt/data1/hibernate/gradle/databases.gradle`, + ); err != nil { + continue + } + + break + } + + t.Status("building hibernate (without tests)") + for attempt, r := 0, retry.StartWithCtx(ctx, opts); r.Next(); { + if ctx.Err() != nil { + return + } + if c.t.Failed() { + return + } + attempt++ + c.l.Printf("attempt %d - building hibernate", attempt) + // Build hibernate and run a single test, this step involves some + // downloading, so it needs a retry loop as well. Just building was not + // enough as the test libraries are not downloaded unless at least a + // single test is invoked. + if err := c.RunE( + ctx, node, `cd /mnt/data1/hibernate/hibernate-core/ && ./../gradlew test -Pdb=cockroach `+ + `--tests org.hibernate.jdbc.util.BasicFormatterTest.*`, + ); err != nil { + continue + } + break + } t.Status("running hibernate test suite, will take at least 3 hours") // When testing, it is helpful to run only a subset of the tests. To do so