[FAB-2709] Fix CouchDB retry logic

To account for cases when peer and couchdb start at same time,
2 minutes of retries are done. This does not work well for chaincode
execution though. Need to fix it such that peer startup verification
retries more times than endorsement/commit interactions with CouchDB,
ideally configurable.

Add two new CouchDB configuration options in core.yaml

couchDBConfig:
	 couchDBAddress: 127.0.0.1:5984
	 username:
	 password:
	 maxRetries: 3
	 maxRetriesOnStartup: 10

- maxRetries will control the max number of retries for http errors
when connecting with CouchDB.

- maxRetriesOnStartup will be a separate configuration option for
specifying the number of retries during startup.  This needs to be
a larger number than the normal retries.

- Handling of 404 error is already handled correctly

- 500 errors and 4XX errors will not be retried.

- Add a change to VerifyConnection to allow for the maxRetriesOnStartup to
be used for retry logic in handleRequest

Change-Id: Ibdf48444ef0728825a168bf4deebde5fa43f12ac
Signed-off-by: Chris Elder <chris.elder@us.ibm.com>

core/chaincode/chaincodetest.yaml

@@ -428,6 +428,8 @@ ledger:
        couchDBAddress: 127.0.0.1:5984
        username:
        password:
+       maxRetries: 3
+       maxRetriesOnStartup: 10
 
     # historyDatabase - options are true or false
     # Indicates if the history of key updates should be stored in goleveldb

core/chaincode/exectransaction_test.go

@@ -124,8 +124,10 @@ func finitPeer(lis net.Listener, chainIDs ...string) {
 		connectURL := viper.GetString("ledger.state.couchDBConfig.couchDBAddress")
 		username := viper.GetString("ledger.state.couchDBConfig.username")
 		password := viper.GetString("ledger.state.couchDBConfig.password")
+		maxRetries := viper.GetInt("ledger.state.couchDBConfig.maxRetries")
+		maxRetriesOnStartup := viper.GetInt("ledger.state.couchDBConfig.maxRetriesOnStartup")
 
-		couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password)
+		couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
 		db, _ := couchdb.CreateCouchDatabase(*couchInstance, chainID)
 		//drop the test database
 		db.DropDatabase()

core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb.go

@@ -55,7 +55,8 @@ type VersionedDBProvider struct {
 func NewVersionedDBProvider() (*VersionedDBProvider, error) {
 	logger.Debugf("constructing CouchDB VersionedDBProvider")
 	couchDBDef := ledgerconfig.GetCouchDBDefinition()
-	couchInstance, err := couchdb.CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password)
+	couchInstance, err := couchdb.CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password,
+		couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup)
 	if err != nil {
 		return nil, err
 	}

core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test_export.go

@@ -29,6 +29,8 @@ var connectURL = "couchdb:5984"
 var badConnectURL = "couchdb:5990"
 var username = ""
 var password = ""
+var maxRetries = 3
+var maxRetriesOnStartup = 10
 
 // TestVDBEnv provides a couch db backed versioned db for testing
 type TestVDBEnv struct {
@@ -55,7 +57,7 @@ func (env *TestVDBEnv) Cleanup(dbName string) {
 }
 func cleanupDB(dbName string) {
 	//create a new connection
-	couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password)
+	couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
 	db := couchdb.CouchDatabase{CouchInstance: *couchInstance, DBName: dbName}
 	//drop the test database
 	db.DropDatabase()

core/ledger/ledgerconfig/ledger_config.go

@@ -22,25 +22,18 @@ import (
 	"github.com/spf13/viper"
 )
 
-// TODO remove all these config variables, they are never used as defaults
-var stateDatabase = "goleveldb"
-var couchDBAddress = "127.0.0.1:5984"
-var username = ""
-var password = ""
-var historyDatabase = true
-
-var maxBlockFileSize = 0
-
 // CouchDBDef contains parameters
 type CouchDBDef struct {
-	URL      string
-	Username string
-	Password string
+	URL                 string
+	Username            string
+	Password            string
+	MaxRetries          int
+	MaxRetriesOnStartup int
 }
 
 //IsCouchDBEnabled exposes the useCouchDB variable
 func IsCouchDBEnabled() bool {
-	stateDatabase = viper.GetString("ledger.state.stateDatabase")
+	stateDatabase := viper.GetString("ledger.state.stateDatabase")
 	if stateDatabase == "CouchDB" {
 		return true
 	}
@@ -82,11 +75,13 @@ func GetMaxBlockfileSize() int {
 //GetCouchDBDefinition exposes the useCouchDB variable
 func GetCouchDBDefinition() *CouchDBDef {
 
-	couchDBAddress = viper.GetString("ledger.state.couchDBConfig.couchDBAddress")
-	username = viper.GetString("ledger.state.couchDBConfig.username")
-	password = viper.GetString("ledger.state.couchDBConfig.password")
+	couchDBAddress := viper.GetString("ledger.state.couchDBConfig.couchDBAddress")
+	username := viper.GetString("ledger.state.couchDBConfig.username")
+	password := viper.GetString("ledger.state.couchDBConfig.password")
+	maxRetries := viper.GetInt("ledger.state.couchDBConfig.maxRetries")
+	maxRetriesOnStartup := viper.GetInt("ledger.state.couchDBConfig.maxRetriesOnStartup")
 
-	return &CouchDBDef{couchDBAddress, username, password}
+	return &CouchDBDef{couchDBAddress, username, password, maxRetries, maxRetriesOnStartup}
 }
 
 //GetQueryLimit exposes the queryLimit variable