[FAB-16879] Add stack trace to couchdb http errors (#1048)

If there is http error calling couchdb, no context was provided
to higher level code that handles the errors.

This change adds stack trace to the http error at the point that the error
is raised, so that admins can identify what was going on upon hitting the error.
In most places, the higher level error handlers are already printing stack
trace, if available. This change adds the stack trace printing to OpenLeder()
error handling, e.g. if there is an error during recommit lost block path.

Also, for each of the points in ledger code where go routines are used,
if stack trace was added in the go routine error, the stack would only
be available from the point the go routine was called and lower. This
is resolved by also adding stack trace to errors that are returned from
go routines.

Signed-off-by: David Enyeart <enyeart@us.ibm.com>

core/ledger/kvledger/txmgmt/statedb/statecouchdb/batch_util.go

@@ -7,6 +7,8 @@ package statecouchdb
 
 import (
 	"sync"
+
+	"github.com/pkg/errors"
 )
 
 // batch is executed in a separate goroutine.
@@ -40,7 +42,7 @@ func executeBatches(batches []batch) error {
 	}
 	batchWG.Wait()
 	if len(errsChan) > 0 {
-		return <-errsChan
+		return errors.WithStack(<-errsChan)
 	}
 	return nil
 }

core/ledger/util/couchdb/couchdb.go

@@ -1821,7 +1821,7 @@ func (couchInstance *CouchInstance) handleRequest(ctx context.Context, method, d
 
 	//if a golang http error is still present after retries are exhausted, return the error
 	if errResp != nil {
-		return nil, couchDBReturn, errResp
+		return nil, couchDBReturn, errors.Wrap(errResp, "http error calling couchdb")
 	}
 
 	//This situation should not occur according to the golang spec.

core/peer/peer.go

@@ -266,7 +266,7 @@ func Initialize(init func(string), ccp ccprovider.ChaincodeProvider, sccp sysccp
 	for _, cid := range ledgerIds {
 		peerLogger.Infof("Loading chain %s", cid)
 		if ledger, err = ledgermgmt.OpenLedger(cid); err != nil {
-			peerLogger.Errorf("Failed to load ledger %s(%s)", cid, err)
+			peerLogger.Errorf("Failed to load ledger %s(%+v)", cid, err)
 			peerLogger.Debugf("Error while loading ledger %s with message %s. We continue to the next ledger rather than abort.", cid, err)
 			continue
 		}

integration/e2e/health_test.go

@@ -146,7 +146,7 @@ var _ = Describe("Health", func() {
 				statusCode, status = DoHealthCheck(authClient, healthURL)
 				Expect(status.Status).To(Equal("Service Unavailable"))
 				Expect(status.FailedChecks[0].Component).To(Equal("couchdb"))
-				Expect(status.FailedChecks[0].Reason).Should((HavePrefix(fmt.Sprintf("failed to connect to couch db [Head http://%s: dial tcp %s: ", couchAddr, couchAddr))))
+				Expect(status.FailedChecks[0].Reason).Should((HavePrefix(fmt.Sprintf("failed to connect to couch db [http error calling couchdb: Head http://%s: dial tcp %s: ", couchAddr, couchAddr))))
 			})
 		})
 	})