Skip to content

Commit

Permalink
Merge pull request #9832 from gyuho/functional
Browse files Browse the repository at this point in the history
functional/tester: fix trigger snapshot retry
  • Loading branch information
gyuho authored Jun 11, 2018
2 parents f985e51 + 19de2b2 commit 322c0df
Showing 1 changed file with 17 additions and 6 deletions.
23 changes: 17 additions & 6 deletions functional/tester/case.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,18 @@ func (c *caseUntilSnapshot) Inject(clus *Cluster) error {

for i := 0; i < retries; i++ {
lastRev, err = clus.maxRev()
if lastRev == 0 {
clus.lg.Info(
"trigger snapshot RETRY",
zap.Int("retries", i),
zap.Int64("etcd-snapshot-count", snapshotCount),
zap.Int64("start-revision", startRev),
zap.Error(err),
)
time.Sleep(3 * time.Second)
continue
}

// If the number of proposals committed is bigger than snapshot count,
// a new snapshot should have been created.
diff := lastRev - startRev
Expand All @@ -292,12 +304,8 @@ func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
return nil
}

dur := time.Second
if diff < 0 || err != nil {
dur = 3 * time.Second
}
clus.lg.Info(
"trigger snapshot PROGRESS",
"trigger snapshot RETRY",
zap.Int("retries", i),
zap.Int64("committed-entries", diff),
zap.Int64("etcd-snapshot-count", snapshotCount),
Expand All @@ -306,7 +314,10 @@ func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
zap.Duration("took", time.Since(now)),
zap.Error(err),
)
time.Sleep(dur)
time.Sleep(time.Second)
if err != nil {
time.Sleep(2 * time.Second)
}
}

return fmt.Errorf("cluster too slow: only %d commits in %d retries", lastRev-startRev, retries)
Expand Down

0 comments on commit 322c0df

Please sign in to comment.