Skip to content

Commit

Permalink
gemini: Progress indicator
Browse files Browse the repository at this point in the history 
This adds a progress indicator to the tool to reduce logging while the
tool is generating and running tests. Users can disable the indicator
with the '--non-interactive' flag for things like CI runs.

Fixes #14
  • Loading branch information
@penberg
penberg committed Mar 15, 2019
1 parent b53321e commit c2df5f5
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 17 deletions.
74 changes: 57 additions & 17 deletions cmd/gemini/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@ package main
import (
"encoding/json"
"fmt"
"golang.org/x/net/context"
"io/ioutil"
"math/rand"
"os"
"sync"
"time"

"github.com/briandowns/spinner"
"github.com/scylladb/gemini"
"github.com/spf13/cobra"
)
Expand All @@ -25,6 +28,7 @@ var (
verbose bool
mode string
failFast bool
nonInteractive bool
)

const (
Expand All @@ -45,7 +49,11 @@ type Results interface {
Print()
}

type testJob func(*gemini.Schema, gemini.Table, *gemini.Session, gemini.PartitionRange, chan Status, string)
func interactive() bool {
return !nonInteractive
}

type testJob func(context.Context, *sync.WaitGroup, *gemini.Schema, gemini.Table, *gemini.Session, gemini.PartitionRange, chan Status, string)

func (r *Status) Merge(sum *Status) Status {
sum.WriteOps += r.WriteOps
Expand Down Expand Up @@ -140,28 +148,55 @@ func run(cmd *cobra.Command, args []string) {
}

func runJob(f testJob, schema *gemini.Schema, s *gemini.Session, mode string) {
var sp *spinner.Spinner = nil
if interactive() {
spinnerCharSet := []string{"|", "/", "-", "\\"}
sp = spinner.New(spinnerCharSet, 1*time.Second)
sp.Color("black")
sp.Start()
}
c := make(chan Status)
minRange := 0
maxRange := pkNumberPerThread

var wg sync.WaitGroup
workerCtx, cancelWorkers := context.WithCancel(context.Background())
for _, table := range schema.Tables {
for i := 0; i < concurrency; i++ {
p := gemini.PartitionRange{Min: minRange + i*maxRange, Max: maxRange + i*maxRange}
go f(schema, table, s, p, c, mode)
wg.Add(1)
go f(workerCtx, &wg, schema, table, s, p, c, mode)
}
}

var testRes Status
for i := 0; i < concurrency*len(schema.Tables); i++ {
res := <-c
testRes = res.Merge(&testRes)
if testRes.ReadErrors > 0 {
testRes.PrintResult()
fmt.Println("Error in data validation. Exiting.")
os.Exit(1)

reporterCtx, cancelReporter := context.WithCancel(context.Background())
go func() {
for {
select {
case <-reporterCtx.Done():
return
case res := <-c:
testRes = res.Merge(&testRes)
sp.Suffix = fmt.Sprintf(" Running Gemini... %v", testRes)
if testRes.ReadErrors > 0 {
testRes.PrintResult()
fmt.Println("Error in data validation. Exiting.")
cancelWorkers()
return
}
break
}
}
}
}()

wg.Wait()
cancelReporter()

if interactive() {
sp.Stop()
}
testRes.PrintResult()
}

Expand Down Expand Up @@ -197,10 +232,16 @@ func validationJob(schema *gemini.Schema, table gemini.Table, s *gemini.Session,
}
}

func Job(schema *gemini.Schema, table gemini.Table, s *gemini.Session, p gemini.PartitionRange, c chan Status, mode string) {
func Job(ctx context.Context, wg *sync.WaitGroup, schema *gemini.Schema, table gemini.Table, s *gemini.Session, p gemini.PartitionRange, c chan Status, mode string) {
defer wg.Done()
testStatus := Status{}

for i := 0; i < maxTests; i++ {
select {
case <-ctx.Done():
return
default:
}
switch mode {
case writeMode:
mutationJob(schema, table, s, p, &testStatus)
Expand All @@ -215,12 +256,10 @@ func Job(schema *gemini.Schema, table gemini.Table, s *gemini.Session, p gemini.
}
}

threadNum := p.Min / pkNumberPerThread
if i%1000 == 0 && threadNum == rand.Intn(concurrency) {
fmt.Printf("thread %v: ", threadNum)
fmt.Println(testStatus)
if i%1000 == 0 {
c <- testStatus
testStatus = Status{}
}

if failFast && testStatus.ReadErrors > 0 {
break
}
Expand Down Expand Up @@ -252,4 +291,5 @@ func init() {
rootCmd.Flags().BoolVarP(&dropSchema, "drop-schema", "d", false, "Drop schema before starting tests run")
rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose output during test run")
rootCmd.Flags().BoolVarP(&failFast, "fail-fast", "f", false, "Stop on the first failure")
rootCmd.Flags().BoolVarP(&nonInteractive, "non-interactive", "", false, "Run in non-interactive mode (disable progress indicator)")
}
5 changes: 5 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,17 @@ go 1.12
require (
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/briandowns/spinner v0.0.0-20190311160019-998b3556fb3f
github.com/fatih/color v1.7.0 // indirect
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4
github.com/google/go-cmp v0.2.0
github.com/google/uuid v1.1.1
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/mattn/go-colorable v0.1.1 // indirect
github.com/mattn/go-isatty v0.0.6 // indirect
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.3 // indirect
github.com/stretchr/testify v1.3.0 // indirect
golang.org/x/net v0.0.0-20190313082753-5c2c250b6a70
)
16 changes: 16 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,12 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/briandowns/spinner v0.0.0-20190311160019-998b3556fb3f h1:MABudtpeYW3sx6sh8sy8k8nPRIQTTvGUOvmUbdvlIGM=
github.com/briandowns/spinner v0.0.0-20190311160019-998b3556fb3f/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4 h1:vF83LI8tAakwEwvWZtrIEx7pOySacl2TOxx6eXk4ePo=
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0=
github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk=
Expand All @@ -21,6 +25,11 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.6 h1:SrwhHcpV4nWrMGdNcC2kXpMfcBVYGDuTArqyhocJgvA=
github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
Expand All @@ -30,5 +39,12 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190313082753-5c2c250b6a70 h1:0OwHPyvXNyZS9VW4XXoGkWOwhrMN52Y4n/gSxvJOgj0=
golang.org/x/net v0.0.0-20190313082753-5c2c250b6a70/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=

0 comments on commit c2df5f5

Please sign in to comment.