From ec79f2a9879cbc0b1cd2520c406eedffbcb14079 Mon Sep 17 00:00:00 2001 From: Matt Fellows Date: Mon, 21 Jun 2021 14:33:18 +1000 Subject: [PATCH] feat: improve consumer error reporting --- MIGRATION.md | 2 +- consumer/http.go | 113 ++++++++++++++++++++++++++++++++++---- docs/consumer.md | 2 +- examples/basic_test.go | 2 +- examples/consumer_test.go | 8 +-- 5 files changed, 110 insertions(+), 17 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 06518234c..3f8136ae0 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -31,7 +31,7 @@ The following package exposes aliases for the most commonly used interfaces for #### Primary Interface - `dsl.Pact` was the primary interface. This is now replaced with `NewPactV2` and the `NewPactV3` methods, which will return you a builder for the corresponding specification. -- `Verify` is now `ExecuteTest` to avoid ambiguity with provider side verification. +- `Verify` is now `ExecuteTest` to avoid ambiguity with provider side verification. It also accepts a `*testing.T` argument, to improve error reporting and resolution. These are available in consumer package: `"github.com/pact-foundation/pact-go/v2/consumer"` diff --git a/consumer/http.go b/consumer/http.go index 673a5081d..3355f7e1d 100644 --- a/consumer/http.go +++ b/consumer/http.go @@ -15,7 +15,12 @@ import ( "log" "os" "path/filepath" + "runtime" + "strings" + "testing" "time" + "unicode" + "unicode/utf8" native "github.com/pact-foundation/pact-go/v2/internal/native/mockserver" logging "github.com/pact-foundation/pact-go/v2/log" @@ -125,7 +130,7 @@ func (p *httpMockProvider) configure() error { // ExecuteTest runs the current test case against a Mock Service. // Will cleanup interactions between tests within a suite // and write the pact file if successful -func (p *httpMockProvider) ExecuteTest(integrationTest func(MockServerConfig) error) error { +func (p *httpMockProvider) ExecuteTest(t *testing.T, integrationTest func(MockServerConfig) error) error { log.Println("[DEBUG] pact verify") var err error @@ -153,7 +158,7 @@ func (p *httpMockProvider) ExecuteTest(integrationTest func(MockServerConfig) er }) res, mismatches := p.mockserver.Verify(p.config.Port, p.config.PactDir) - p.displayMismatches(mismatches) + p.displayMismatches(t, mismatches) if err != nil { return err @@ -177,18 +182,25 @@ func (p *httpMockProvider) reset() { p.configure() } -// TODO: pretty print this to make it really easy to understand the problems +// TODO: improve / pretty print this to make it really easy to understand the problems // See existing Pact/Ruby code examples -func (p *httpMockProvider) displayMismatches(mismatches []native.MismatchedRequest) { +func (p *httpMockProvider) displayMismatches(t *testing.T, mismatches []native.MismatchedRequest) { if len(mismatches) > 0 { + + if len(callerInfo()) > 0 { + fmt.Printf("\n\n%s:\n", callerInfo()[len(callerInfo())-1]) + } + fmt.Println("\tPact Verification Failed for:", t.Name()) + fmt.Println() + fmt.Println("\t\tDiff:") log.Println("[INFO] pact validation failed, errors: ") for _, m := range mismatches { formattedRequest := fmt.Sprintf("%s %s", m.Request.Method, m.Request.Path) switch m.Type { case "missing-request": - fmt.Printf("Expected request to: %s, but did not receive one\n", formattedRequest) + fmt.Printf("\t\texpected: \t%s (Expected request that was not received)\n", formattedRequest) case "request-not-found": - fmt.Printf("Unexpected request was received: %s\n", formattedRequest) + fmt.Printf("\t\tactual: \t%s (Unexpected request was received)\n", formattedRequest) default: // TODO: } @@ -196,14 +208,95 @@ func (p *httpMockProvider) displayMismatches(mismatches []native.MismatchedReque for _, detail := range m.Mismatches { switch detail.Type { case "HeaderMismatch": - fmt.Printf("Comparing Header: '%s'\n", detail.Key) - fmt.Println(detail.Mismatch) - fmt.Println("Expected:", detail.Expected) - fmt.Println("Actual:", detail.Actual) + fmt.Printf("\t\t\tComparing Header: '%s'\n", detail.Key) + fmt.Println("\t\t\t", detail.Mismatch) + fmt.Println("\t\t\texpected: \t", detail.Expected) + fmt.Println("\t\t\tactual: \t", detail.Actual) + case "BodyMismatch": + fmt.Printf("\t\t\t%s\n", detail.Mismatch) + fmt.Println("\t\t\texpected:\t", detail.Expected) + fmt.Println("\t\t\tactual:\t\t", detail.Actual) } } } + fmt.Println() + fmt.Println() + } +} + +// Stolen from "github.com/stretchr/testify/assert" +func callerInfo() []string { + + var pc uintptr + var ok bool + var file string + var line int + var name string + + callers := []string{} + for i := 0; ; i++ { + pc, file, line, ok = runtime.Caller(i) + if !ok { + // The breaks below failed to terminate the loop, and we ran off the + // end of the call stack. + break + } + + // This is a huge edge case, but it will panic if this is the case, see #180 + if file == "" { + break + } + + f := runtime.FuncForPC(pc) + if f == nil { + break + } + name = f.Name() + + // testing.tRunner is the standard library function that calls + // tests. Subtests are called directly by tRunner, without going through + // the Test/Benchmark/Example function that contains the t.Run calls, so + // with subtests we should break when we hit tRunner, without adding it + // to the list of callers. + if name == "testing.tRunner" { + break + } + + parts := strings.Split(file, "/") + file = parts[len(parts)-1] + if len(parts) > 1 { + dir := parts[len(parts)-2] + if (dir != "assert" && dir != "mock" && dir != "require") || file == "mock_test.go" { + callers = append(callers, fmt.Sprintf("%s:%d", file, line)) + } + } + + // Drop the package + segments := strings.Split(name, ".") + name = segments[len(segments)-1] + if isTest(name, "Test") || + isTest(name, "Benchmark") || + isTest(name, "Example") { + break + } + } + + return callers +} + +// Stolen from the `go test` tool. +// isTest tells whether name looks like a test (or benchmark, according to prefix). +// It is a Test (say) if there is a character after Test that is not a lower-case letter. +// We don't want TesticularCancer. +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { // "Test" is ok + return true } + r, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(r) } // writePact may be called after each interaction with a mock server is completed diff --git a/docs/consumer.md b/docs/consumer.md index 7331002aa..797e7c1cc 100644 --- a/docs/consumer.md +++ b/docs/consumer.md @@ -56,7 +56,7 @@ func TestProductAPIClient(t *testing.T) { WithBodyMatch(&Product{}) // This uses struct tags for matchers // Act: test our API client behaves correctly - err = mockProvider.ExecuteTest(func(config MockServerConfig) error { + err = mockProvider.ExecuteTest(t, func(config MockServerConfig) error { // Initialise the API client and point it at the Pact mock server // Pact spins up a dedicated mock server for each test client := newClient(config.Host, config.Port) diff --git a/examples/basic_test.go b/examples/basic_test.go index 0d32b9b7c..36d89f6d0 100644 --- a/examples/basic_test.go +++ b/examples/basic_test.go @@ -29,7 +29,7 @@ func TestProductAPIClient(t *testing.T) { WithBodyMatch(&Product{}) // Act: test our API client behaves correctly - err = mockProvider.ExecuteTest(func(config MockServerConfig) error { + err = mockProvider.ExecuteTest(t, func(config MockServerConfig) error { // Initialise the API client and point it at the Pact mock server client := newClient(config.Host, config.Port) diff --git a/examples/consumer_test.go b/examples/consumer_test.go index 50b5bbdc4..8df3ff7be 100644 --- a/examples/consumer_test.go +++ b/examples/consumer_test.go @@ -56,7 +56,7 @@ func TestConsumerV2(t *testing.T) { }) // Execute pact test - err = mockProvider.ExecuteTest(test) + err = mockProvider.ExecuteTest(t, test) assert.NoError(t, err) } @@ -87,7 +87,7 @@ func TestConsumerV2_Match(t *testing.T) { WithBodyMatch(&User{}) // Execute pact test - err = mockProvider.ExecuteTest(test) + err = mockProvider.ExecuteTest(t, test) assert.NoError(t, err) } @@ -144,7 +144,7 @@ func TestConsumerV3(t *testing.T) { }) // Execute pact test - err = mockProvider.ExecuteTest(test) + err = mockProvider.ExecuteTest(t, test) assert.NoError(t, err) } @@ -193,7 +193,7 @@ func TestConsumerV2AllInOne(t *testing.T) { }) // Execute pact test - err = mockProvider.ExecuteTest(legacyTest) + err = mockProvider.ExecuteTest(t, legacyTest) assert.NoError(t, err) }