diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f96e158..1b90a83 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -171,3 +171,28 @@ jobs: env_vars: OS,GOLANG name: codecov-umbrella fail_ci_if_error: false + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + golang: + - 1.15.1 + env: + OS: ubuntu-latest + GOLANG: ${{ matrix.golang }} + steps: + - uses: actions/checkout@v2 + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.golang }} + - uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ matrix.golang }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ matrix.golang }}- + - name: Compile the project + run: make go.install + - name: Run integration tests + run: make integration diff --git a/Makefile b/Makefile index 0930f7b..006cd15 100644 --- a/Makefile +++ b/Makefile @@ -4,3 +4,6 @@ GOBINS ?= . NPM_PACKAGES ?= . include rules.mk + +integration: install + cd examples && make integration diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 0000000..b660ec9 --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,14 @@ +integration: + # test with testman + testman test -run ^TestStable ./... + @# FIXME: test unstable tests + @# FIXME: test broken tests + + # test with default tools + go install moul.io/retry + go test -run ^TestStable -count=20 ./... >/dev/null # should always work + retry -m=5 --interval=0 -- "(go test -run ^TestBroken -count=1 ./... >/dev/null)" && exit 1 || exit 0 # should always fail + retry -m=50 --interval=0 -- "(go test -run ^TestUnstable -count 1 ./... >/dev/null)" # should work at least 1/50 + go mod tidy + + @echo "SUCCESS." diff --git a/examples/go.mod b/examples/go.mod new file mode 100644 index 0000000..786d248 --- /dev/null +++ b/examples/go.mod @@ -0,0 +1,5 @@ +module moul.io/testman/examples + +go 1.15 + +require moul.io/srand v1.5.0 diff --git a/examples/go.sum b/examples/go.sum new file mode 100644 index 0000000..1b4d0b5 --- /dev/null +++ b/examples/go.sum @@ -0,0 +1,2 @@ +moul.io/srand v1.5.0 h1:xGelp9uiK52NClaWCvQWHvVAXzhVrWDA5NiwnGp2JkU= +moul.io/srand v1.5.0/go.mod h1:P2uaZB+GFstFNo8sEj6/U8FRV1n25kD0LLckFpJ+qvc= diff --git a/examples/testpkg/testpkg.go b/examples/testpkg/testpkg.go new file mode 100644 index 0000000..3ab2a41 --- /dev/null +++ b/examples/testpkg/testpkg.go @@ -0,0 +1,21 @@ +package testpkg + +import ( + "fmt" + "math/rand" +) + +func AlwaysSucceed() error { + return nil +} + +func AlwaysFailing() error { + return fmt.Errorf("hope is the key to life") +} + +func MaySucceed() error { + if rand.Intn(3) != 0 { + return fmt.Errorf("oops, no luck, try again") + } + return nil +} diff --git a/examples/testpkg/testpkg_test.go b/examples/testpkg/testpkg_test.go new file mode 100644 index 0000000..9d4fd13 --- /dev/null +++ b/examples/testpkg/testpkg_test.go @@ -0,0 +1,33 @@ +package testpkg + +import ( + "fmt" + "math/rand" + "testing" + + "moul.io/srand" +) + +func ExampleAlwaysSucceed() { + fmt.Println(AlwaysSucceed()) + // Output: +} + +func TestStableAlwaysSucceed(t *testing.T) { + if err := AlwaysSucceed(); err != nil { + t.Errorf("expect no error, got %v", err) + } +} + +func TestUnstableMaySucceed(t *testing.T) { + rand.Seed(srand.Fast()) + if err := MaySucceed(); err != nil { + t.Errorf("expect no error, got %v", err) + } +} + +func TestBrokenAlwaysFailing(t *testing.T) { + if err := AlwaysFailing(); err != nil { + t.Errorf("expect no error, got %v", err) + } +} diff --git a/go.mod b/go.mod index c98339d..9562ece 100644 --- a/go.mod +++ b/go.mod @@ -5,5 +5,6 @@ go 1.13 require ( github.com/peterbourgon/ff/v3 v3.0.0 go.uber.org/goleak v1.1.10 + moul.io/godev v1.7.0 moul.io/motd v1.0.0 ) diff --git a/go.sum b/go.sum index fe86918..7d44bc6 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,10 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -26,6 +30,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -37,7 +42,11 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22 h1:0efs3hwEZhFKsCoP8l6dDB1AZWMgnEl3yWXWRZTOaEA= +gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= moul.io/banner v1.0.1 h1:+WsemGLhj2pOajw2eR5VYjLhOIqs0XhIRYchzTyMLk0= moul.io/banner v1.0.1/go.mod h1:XwvIGKkhKRKyN1vIdmR5oaKQLIkMhkMqrsHpS94QzAU= +moul.io/godev v1.7.0 h1:PgnL7BsCQPPjKwu9V0oxIVm2MyZAHAN2sl0S3+E37U0= +moul.io/godev v1.7.0/go.mod h1:5lgSpI1oH7xWpLl2Ew/Nsgk8DiNM6FzN9WV9+lgW8RQ= moul.io/motd v1.0.0 h1:Trk4fPibDfPJf2iCBSQC8ws7Q02sMwivQdVEFAjCPto= moul.io/motd v1.0.0/go.mod h1:39rvZ0lC2oRhHDY2VoPyZ8r70VKqeJye3QAxjeLDJso= diff --git a/main.go b/main.go index 4ec4a08..25762de 100644 --- a/main.go +++ b/main.go @@ -9,10 +9,12 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "time" "github.com/peterbourgon/ff/v3/ffcli" + "moul.io/godev" "moul.io/motd" ) @@ -28,15 +30,15 @@ func main() { } func run(args []string) error { - opts = Opts{ - verbose: false, - } - + // flags testFlags := flag.NewFlagSet("testman test", flag.ExitOnError) - //testFlags.BoolVar(&opts.continueOnFailure, "continue-on-failure", opts.continueOnFailure, "Continue on failure") - testFlags.BoolVar(&opts.verbose, "v", opts.verbose, "verbose") + testFlags.BoolVar(&opts.Verbose, "v", false, "verbose") + testFlags.StringVar(&opts.Run, "run", "^(Test|Example)", "regex to filter out tests and examples") + //testFlags.IntVar(&opts.Retry, "retry", 0, "fail after N retries") + //testFlags.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "max duration allowed to run the whole suite") listFlags := flag.NewFlagSet("testman list", flag.ExitOnError) - listFlags.BoolVar(&opts.verbose, "v", opts.verbose, "verbose") + listFlags.BoolVar(&opts.Verbose, "v", false, "verbose") + listFlags.StringVar(&opts.Run, "run", "^(Test|Example)", "regex to filter out tests and examples") root := &ffcli.Command{ ShortUsage: "testman [flags]", @@ -85,6 +87,9 @@ func runList(ctx context.Context, args []string) error { if err != nil { return err } + if len(tests) == 0 { + continue + } fmt.Println(pkg.ImportPath) for _, test := range tests { @@ -99,7 +104,7 @@ func runTest(ctx context.Context, args []string) error { return flag.ErrHelp } preRun() - + log.Printf("runTest opts=%s args=%s", godev.JSON(opts), godev.JSON(args)) start := time.Now() // list packages @@ -122,6 +127,9 @@ func runTest(ctx context.Context, args []string) error { if err != nil { return err } + if len(tests) == 0 { + continue + } pkgStart := time.Now() // compile test binary @@ -138,7 +146,7 @@ func runTest(ctx context.Context, args []string) error { "-test.count=1", "-test.timeout=300s", } - if opts.verbose { + if opts.Verbose { args = append(args, "-test.v") } args = append(args, "-test.run", fmt.Sprintf("^%s$", test)) @@ -146,8 +154,8 @@ func runTest(ctx context.Context, args []string) error { log.Println(cmd.String()) out, err := cmd.CombinedOutput() if err != nil { - fmt.Printf("FAIL\t%s\t[compile error: %v]\n", pkg.ImportPath, err) - if opts.verbose { + fmt.Printf("FAIL\t%s.%s\t[compile error: %v]\n", pkg.ImportPath, test, err) + if opts.Verbose { fmt.Println(string(out)) } isPackageOK = false @@ -167,7 +175,7 @@ func runTest(ctx context.Context, args []string) error { } func preRun() { - if !opts.verbose { + if !opts.Verbose { log.SetOutput(ioutil.Discard) } } @@ -204,6 +212,15 @@ func listDirTests(dir string) ([]string, error) { if strings.HasPrefix(line, "ok ") { continue } + if opts.Run != "" { + matched, err := regexp.MatchString(opts.Run, line) + if err != nil { + return nil, err + } + if !matched { + continue + } + } tests = append(tests, line) } return tests, nil @@ -239,11 +256,11 @@ type Package struct { } type Opts struct { - verbose bool - // run - // timeout + Verbose bool + Run string + // Timeout time.Duration + // Retry int // c // debug - // retries // continueOnFailure vs failFast }