Merge pull request #7 from moul/dev/moul/continue-on-error

feat: testman test -continue-on-error

.golangci.yml

@@ -39,7 +39,7 @@ linters:
     - govet
     - ineffassign
     - interfacer
-    - maligned
+    #- maligned
     - misspell
     - nakedret
     - prealloc

examples/Makefile

@@ -1,8 +1,12 @@
+TMPDIR ?= /tmp/testman-integration
+
 integration:
 	# test with testman
+	@mkdir -p $(TMPDIR)
 	testman test -timeout=10s -run ^TestStable ./...  # should always work
 	testman test -timeout=60s -run ^TestUnstable -retry=50 ./...  # should work at least 1/50
-	@# FIXME: test broken tests
+	(testman test -timeout=10s -continue-on-error -retry=10 ./...; echo "EXIT CODE: $$?") | tee $(TMPDIR)/got.continue-on-error.log	  # should fail, but also run all tests
+	diff ./testdata/expected.continue-on-error.log $(TMPDIR)/got.continue-on-error.log
 
 	# test with default tools
 	go install moul.io/retry
@@ -12,3 +16,4 @@ integration:
 	go mod tidy
 
 	@echo "SUCCESS."
+	@rm -rf $(TMPDIR)

examples/testdata/expected.continue-on-error.log

@@ -0,0 +1,5 @@
+ok	moul.io/testman/examples/testpkg.TestStableAlwaysSucceed
+ok	moul.io/testman/examples/testpkg.TestUnstableMaySucceed
+FAIL	moul.io/testman/examples/testpkg.TestBrokenAlwaysFailing	[test error: exit status 1]
+ok	moul.io/testman/examples/testpkg.ExampleAlwaysSucceed
+EXIT CODE: 1

main.go

@@ -35,7 +35,8 @@ func run(args []string) error {
 	testFlags.BoolVar(&opts.Verbose, "v", false, "verbose")
 	testFlags.StringVar(&opts.Run, "run", "^(Test|Example)", "regex to filter out tests and examples")
 	testFlags.IntVar(&opts.Retry, "retry", 0, "fail after N retries")
-	testFlags.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "max duration allowed to run the whole suite")
+	testFlags.DurationVar(&opts.Timeout, "timeout", 0, "max duration allowed to run the whole suite")
+	testFlags.BoolVar(&opts.ContinueOnError, "continue-on-error", false, "continue on error (but still fails at the end)")
 	listFlags := flag.NewFlagSet("testman list", flag.ExitOnError)
 	listFlags.BoolVar(&opts.Verbose, "v", false, "verbose")
 	listFlags.StringVar(&opts.Run, "run", "^(Test|Example)", "regex to filter out tests and examples")
@@ -298,11 +299,12 @@ type Package struct {
 }
 
 type Opts struct {
-	Verbose bool
-	Run     string
-	Timeout time.Duration
-	Retry   int
-	TmpDir  string
+	Verbose         bool
+	Run             string
+	Timeout         time.Duration
+	Retry           int
+	TmpDir          string
+	ContinueOnError bool
 	// c
 	// debug
 	// continueOnFailure vs failFast