feat: add -run flag + setup integration test

.github/workflows/go.yml

@@ -171,3 +171,28 @@ jobs:
           env_vars: OS,GOLANG
           name: codecov-umbrella
           fail_ci_if_error: false
+  integration-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        golang:
+          - 1.15.1
+    env:
+      OS: ubuntu-latest
+      GOLANG: ${{ matrix.golang }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.golang }}
+      - uses: actions/cache@v1
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ matrix.golang }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ matrix.golang }}-
+      - name: Compile the project
+        run: make go.install
+      - name: Run integration tests
+        run: make integration

Makefile

@@ -4,3 +4,6 @@ GOBINS ?=	.
 NPM_PACKAGES ?=	.
 
 include rules.mk
+
+integration: install
+	cd examples && make integration

examples/Makefile

@@ -0,0 +1,14 @@
+integration:
+	# test with testman
+	testman test -run ^TestStable ./...
+	@# FIXME: test unstable tests
+	@# FIXME: test broken tests
+
+	# test with default tools
+	go install moul.io/retry
+	go test -run ^TestStable -count=20 ./... >/dev/null                                                     # should always work
+	retry -m=5  --interval=0 -- "(go test -run ^TestBroken -count=1 ./... >/dev/null)" && exit 1 || exit 0  # should always fail
+	retry -m=50 --interval=0 -- "(go test -run ^TestUnstable -count 1 ./... >/dev/null)"                    # should work at least 1/50
+	go mod tidy
+
+	@echo "SUCCESS."

examples/testpkg/testpkg.go

@@ -0,0 +1,21 @@
+package testpkg
+
+import (
+	"fmt"
+	"math/rand"
+)
+
+func AlwaysSucceed() error {
+	return nil
+}
+
+func AlwaysFailing() error {
+	return fmt.Errorf("hope is the key to life")
+}
+
+func MaySucceed() error {
+	if rand.Intn(3) != 0 {
+		return fmt.Errorf("oops, no luck, try again")
+	}
+	return nil
+}

examples/testpkg/testpkg_test.go

@@ -0,0 +1,33 @@
+package testpkg
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"moul.io/srand"
+)
+
+func ExampleAlwaysSucceed() {
+	fmt.Println(AlwaysSucceed())
+	// Output: <nil>
+}
+
+func TestStableAlwaysSucceed(t *testing.T) {
+	if err := AlwaysSucceed(); err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+}
+
+func TestUnstableMaySucceed(t *testing.T) {
+	rand.Seed(srand.Fast())
+	if err := MaySucceed(); err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+}
+
+func TestBrokenAlwaysFailing(t *testing.T) {
+	if err := AlwaysFailing(); err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+}

main.go

@@ -9,10 +9,12 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
+	"moul.io/godev"
 	"moul.io/motd"
 )
 
@@ -28,15 +30,15 @@ func main() {
 }
 
 func run(args []string) error {
-	opts = Opts{
-		verbose: false,
-	}
-
+	// flags
 	testFlags := flag.NewFlagSet("testman test", flag.ExitOnError)
-	//testFlags.BoolVar(&opts.continueOnFailure, "continue-on-failure", opts.continueOnFailure, "Continue on failure")
-	testFlags.BoolVar(&opts.verbose, "v", opts.verbose, "verbose")
+	testFlags.BoolVar(&opts.Verbose, "v", false, "verbose")
+	testFlags.StringVar(&opts.Run, "run", "^(Test|Example)", "regex to filter out tests and examples")
+	//testFlags.IntVar(&opts.Retry, "retry", 0, "fail after N retries")
+	//testFlags.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "max duration allowed to run the whole suite")
 	listFlags := flag.NewFlagSet("testman list", flag.ExitOnError)
-	listFlags.BoolVar(&opts.verbose, "v", opts.verbose, "verbose")
+	listFlags.BoolVar(&opts.Verbose, "v", false, "verbose")
+	listFlags.StringVar(&opts.Run, "run", "^(Test|Example)", "regex to filter out tests and examples")
 
 	root := &ffcli.Command{
 		ShortUsage: "testman <subcommand> [flags]",
@@ -85,6 +87,9 @@ func runList(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
+		if len(tests) == 0 {
+			continue
+		}
 
 		fmt.Println(pkg.ImportPath)
 		for _, test := range tests {
@@ -99,7 +104,7 @@ func runTest(ctx context.Context, args []string) error {
 		return flag.ErrHelp
 	}
 	preRun()
-
+	log.Printf("runTest opts=%s args=%s", godev.JSON(opts), godev.JSON(args))
 	start := time.Now()
 
 	// list packages
@@ -122,6 +127,9 @@ func runTest(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
+		if len(tests) == 0 {
+			continue
+		}
 
 		pkgStart := time.Now()
 		// compile test binary
@@ -138,16 +146,16 @@ func runTest(ctx context.Context, args []string) error {
 				"-test.count=1",
 				"-test.timeout=300s",
 			}
-			if opts.verbose {
+			if opts.Verbose {
 				args = append(args, "-test.v")
 			}
 			args = append(args, "-test.run", fmt.Sprintf("^%s$", test))
 			cmd := exec.Command(bin, args...)
 			log.Println(cmd.String())
 			out, err := cmd.CombinedOutput()
 			if err != nil {
-				fmt.Printf("FAIL\t%s\t[compile error: %v]\n", pkg.ImportPath, err)
-				if opts.verbose {
+				fmt.Printf("FAIL\t%s.%s\t[compile error: %v]\n", pkg.ImportPath, test, err)
+				if opts.Verbose {
 					fmt.Println(string(out))
 				}
 				isPackageOK = false
@@ -167,7 +175,7 @@ func runTest(ctx context.Context, args []string) error {
 }
 
 func preRun() {
-	if !opts.verbose {
+	if !opts.Verbose {
 		log.SetOutput(ioutil.Discard)
 	}
 }
@@ -204,6 +212,15 @@ func listDirTests(dir string) ([]string, error) {
 		if strings.HasPrefix(line, "ok ") {
 			continue
 		}
+		if opts.Run != "" {
+			matched, err := regexp.MatchString(opts.Run, line)
+			if err != nil {
+				return nil, err
+			}
+			if !matched {
+				continue
+			}
+		}
 		tests = append(tests, line)
 	}
 	return tests, nil
@@ -239,11 +256,11 @@ type Package struct {
 }
 
 type Opts struct {
-	verbose bool
-	// run
-	// timeout
+	Verbose bool
+	Run     string
+	// Timeout time.Duration
+	// Retry   int
 	// c
 	// debug
-	// retries
 	// continueOnFailure vs failFast
 }