From 668b4c308cf7593944762e2ce0021e852face606 Mon Sep 17 00:00:00 2001 From: Adnan Choudhury Date: Thu, 9 Mar 2017 21:28:57 -0500 Subject: [PATCH] [FAB-2662] Implement CouchDB docker config https://jira.hyperledger.org/browse/FAB-2662 Changing the CouchDB dockers used in fabric to use recommended configuration (couchdb local.ini). Include config comments in local.ini stating how to enable user security. Also, making sure that all CouchDB unit-tests follow the same procedure for setting up address and login credentials. Including test comments stating how to run tests against CouchDB with user security enabled. Change-Id: I975d04d757d0371c8db03acf0ddcf92c01c35f8c Signed-off-by: Adnan Choudhury Signed-off-by: denyeart --- core/chaincode/exectransaction_test.go | 2 +- .../statedb/statecouchdb/statecouchdb_test.go | 18 +++- .../statecouchdb/statecouchdb_test_export.go | 15 +-- .../txmgmt/txmgr/commontests/pkg_test.go | 8 ++ core/ledger/util/couchdb/couchdb_test.go | 93 ++++++++++++------- core/ledger/util/couchdb/couchdbutil_test.go | 3 +- images/couchdb/local.ini | 44 +++++++++ 7 files changed, 138 insertions(+), 45 deletions(-) diff --git a/core/chaincode/exectransaction_test.go b/core/chaincode/exectransaction_test.go index 9fe7984520f..e3d71a05c92 100644 --- a/core/chaincode/exectransaction_test.go +++ b/core/chaincode/exectransaction_test.go @@ -130,7 +130,7 @@ func finitPeer(lis net.Listener, chainIDs ...string) { requestTimeout := viper.GetDuration("ledger.state.couchDBConfig.requestTimeout") couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) - db, _ := couchdb.CreateCouchDatabase(*couchInstance, chainID) + db := couchdb.CouchDatabase{CouchInstance: *couchInstance, DBName: chainID} //drop the test database db.DropDatabase() diff --git a/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test.go b/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test.go index 3defdea2ab7..b9dd3df3b6d 100644 --- a/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test.go +++ b/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test.go @@ -19,6 +19,7 @@ package statecouchdb import ( "os" "testing" + "time" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb" @@ -31,15 +32,28 @@ import ( func TestMain(m *testing.M) { - //call a helper method to load the core.yaml, will be used to detect if CouchDB is enabled + // Read the core.yaml file for default config. ledgertestutil.SetupCoreYAMLConfig("./../../../../../../peer") + viper.Set("peer.fileSystemPath", "/tmp/fabric/ledgertests/kvledger/txmgmt/statedb/statecouchdb") + + // Switch to CouchDB viper.Set("ledger.state.stateDatabase", "CouchDB") // both vagrant and CI have couchdb configured at host "couchdb" viper.Set("ledger.state.couchDBConfig.couchDBAddress", "couchdb:5984") - + // Replace with correct username/password such as + // admin/admin if user security is enabled on couchdb. + viper.Set("ledger.state.couchDBConfig.username", "") + viper.Set("ledger.state.couchDBConfig.password", "") + viper.Set("ledger.state.couchDBConfig.maxRetries", 3) + viper.Set("ledger.state.couchDBConfig.maxRetriesOnStartup", 10) + viper.Set("ledger.state.couchDBConfig.requestTimeout", time.Second*20) + + //run the actual test result := m.Run() + + //revert to default goleveldb viper.Set("ledger.state.stateDatabase", "goleveldb") os.Exit(result) } diff --git a/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test_export.go b/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test_export.go index 0fc6fc7d4ac..54df05b0d2c 100644 --- a/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test_export.go +++ b/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test_export.go @@ -19,20 +19,13 @@ package statecouchdb import ( "strings" "testing" - "time" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb" + "github.com/hyperledger/fabric/core/ledger/ledgerconfig" "github.com/hyperledger/fabric/core/ledger/util/couchdb" ) -//Basic setup to test couch -var connectURL = "couchdb:5984" -var badConnectURL = "couchdb:5990" -var username = "" -var password = "" -var maxRetries = 3 -var maxRetriesOnStartup = 10 -var connectionTimeout = time.Second * 60 +const badConnectURL = "couchdb:5990" // TestVDBEnv provides a couch db backed versioned db for testing type TestVDBEnv struct { @@ -59,7 +52,9 @@ func (env *TestVDBEnv) Cleanup(dbName string) { } func cleanupDB(dbName string) { //create a new connection - couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, connectionTimeout) + couchDBDef := ledgerconfig.GetCouchDBDefinition() + couchInstance, _ := couchdb.CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) db := couchdb.CouchDatabase{CouchInstance: *couchInstance, DBName: dbName} //drop the test database db.DropDatabase() diff --git a/core/ledger/kvledger/txmgmt/txmgr/commontests/pkg_test.go b/core/ledger/kvledger/txmgmt/txmgr/commontests/pkg_test.go index 2567605a749..c404065141e 100644 --- a/core/ledger/kvledger/txmgmt/txmgr/commontests/pkg_test.go +++ b/core/ledger/kvledger/txmgmt/txmgr/commontests/pkg_test.go @@ -18,6 +18,7 @@ package commontests import ( "testing" + "time" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb" @@ -106,6 +107,13 @@ func (env *couchDBLockBasedEnv) init(t *testing.T, testLedgerID string) { viper.Set("peer.fileSystemPath", testFilesystemPath) // both vagrant and CI have couchdb configured at host "couchdb" viper.Set("ledger.state.couchDBConfig.couchDBAddress", "couchdb:5984") + // Replace with correct username/password such as + // admin/admin if user security is enabled on couchdb. + viper.Set("ledger.state.couchDBConfig.username", "") + viper.Set("ledger.state.couchDBConfig.password", "") + viper.Set("ledger.state.couchDBConfig.maxRetries", 3) + viper.Set("ledger.state.couchDBConfig.maxRetriesOnStartup", 10) + viper.Set("ledger.state.couchDBConfig.requestTimeout", time.Second*20) testDBEnv := statecouchdb.NewTestVDBEnv(t) testDB, err := testDBEnv.DBProvider.GetDBHandle(testLedgerID) testutil.AssertNoError(t, err, "") diff --git a/core/ledger/util/couchdb/couchdb_test.go b/core/ledger/util/couchdb/couchdb_test.go index e7964069fa3..b2547f9091f 100644 --- a/core/ledger/util/couchdb/couchdb_test.go +++ b/core/ledger/util/couchdb/couchdb_test.go @@ -31,21 +31,17 @@ import ( "github.com/spf13/viper" ) -//Basic setup to test couch -var connectURL = "couchdb:5984" -var badConnectURL = "couchdb:5990" -var username = "" -var password = "" -var maxRetries = 3 -var maxRetriesOnStartup = 10 -var requestTimeout = time.Second * 20 - +const badConnectURL = "couchdb:5990" const updateDocumentConflictError = "conflict" const updateDocumentConflictReason = "Document update conflict." +var couchDBDef *ledgerconfig.CouchDBDef + func cleanup(database string) error { //create a new connection - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) + if err != nil { fmt.Println("Unexpected error", err) return err @@ -68,20 +64,38 @@ type Asset struct { var assetJSON = []byte(`{"asset_name":"marble1","color":"blue","size":"35","owner":"jerry"}`) func TestMain(m *testing.M) { + // Read the core.yaml file for default config. ledgertestutil.SetupCoreYAMLConfig("./../../../../peer") + + // Switch to CouchDB viper.Set("ledger.state.stateDatabase", "CouchDB") + + // both vagrant and CI have couchdb configured at host "couchdb" + viper.Set("ledger.state.couchDBConfig.couchDBAddress", "couchdb:5984") + // Replace with correct username/password such as + // admin/admin if user security is enabled on couchdb. + viper.Set("ledger.state.couchDBConfig.username", "") + viper.Set("ledger.state.couchDBConfig.password", "") + viper.Set("ledger.state.couchDBConfig.maxRetries", 3) + viper.Set("ledger.state.couchDBConfig.maxRetriesOnStartup", 10) + viper.Set("ledger.state.couchDBConfig.requestTimeout", time.Second*20) + + // Create CouchDB definition from config parameters + couchDBDef = ledgerconfig.GetCouchDBDefinition() + + //run the tests result := m.Run() + + //revert to default goleveldb viper.Set("ledger.state.stateDatabase", "goleveldb") os.Exit(result) } func TestDBConnectionDef(t *testing.T) { - //call a helper method to load the core.yaml - ledgertestutil.SetupCoreYAMLConfig("./../../../../peer") - //create a new connection - _, err := CreateConnectionDefinition(connectURL, "", "", maxRetries, maxRetriesOnStartup, requestTimeout) + _, err := CreateConnectionDefinition(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition")) } @@ -89,7 +103,8 @@ func TestDBConnectionDef(t *testing.T) { func TestDBBadConnectionDef(t *testing.T) { //create a new connection - _, err := CreateConnectionDefinition("^^^localhost:5984", "", "", maxRetries, maxRetriesOnStartup, requestTimeout) + _, err := CreateConnectionDefinition("^^^localhost:5984", couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertError(t, err, fmt.Sprintf("Did not receive error when trying to create database connection definition with a bad hostname")) } @@ -105,7 +120,8 @@ func TestDBCreateSaveWithoutRevision(t *testing.T) { if err == nil { //create a new instance and database object - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -131,7 +147,8 @@ func TestDBCreateEnsureFullCommit(t *testing.T) { if err == nil { //create a new instance and database object - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -156,25 +173,29 @@ func TestDBBadDatabaseName(t *testing.T) { if ledgerconfig.IsCouchDBEnabled() { //create a new instance and database object using a valid database name mixed case - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) _, dberr := CreateCouchDatabase(*couchInstance, "testDB") testutil.AssertNoError(t, dberr, fmt.Sprintf("Error when testing a valid database name")) //create a new instance and database object using a valid database name letters and numbers - couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err = CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) _, dberr = CreateCouchDatabase(*couchInstance, "test132") testutil.AssertNoError(t, dberr, fmt.Sprintf("Error when testing a valid database name")) //create a new instance and database object using a valid database name - special characters - couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err = CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) _, dberr = CreateCouchDatabase(*couchInstance, "test1234~!@#$%^&*()[]{}.") testutil.AssertNoError(t, dberr, fmt.Sprintf("Error when testing a valid database name")) //create a new instance and database object using a invalid database name - too long /* - couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err = CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) _, dberr = CreateCouchDatabase(*couchInstance, "A12345678901234567890123456789012345678901234"+ "56789012345678901234567890123456789012345678901234567890123456789012345678901234567890"+ @@ -195,7 +216,8 @@ func TestDBBadConnection(t *testing.T) { if ledgerconfig.IsCouchDBEnabled() { //create a new instance and database object - _, err := CreateCouchInstance(badConnectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + _, err := CreateCouchInstance(badConnectURL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertError(t, err, fmt.Sprintf("Error should have been thrown for a bad connection")) } } @@ -211,7 +233,8 @@ func TestDBCreateDatabaseAndPersist(t *testing.T) { if err == nil { //create a new instance and database object - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -305,7 +328,8 @@ func TestDBRequestTimeout(t *testing.T) { //create a new instance and database object with a timeout that will fail //Also use a maxRetriesOnStartup=3 to reduce the number of retries - _, err := CreateCouchInstance(connectURL, username, password, maxRetries, 3, impossibleTimeout) + _, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, 3, impossibleTimeout) testutil.AssertError(t, err, fmt.Sprintf("Error should have been thown while trying to create a couchdb instance with a connection timeout")) //see if the error message contains the timeout error @@ -327,7 +351,8 @@ func TestDBBadJSON(t *testing.T) { if err == nil { //create a new instance and database object - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -363,7 +388,8 @@ func TestPrefixScan(t *testing.T) { if err == nil { //create a new instance and database object - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -435,7 +461,8 @@ func TestDBSaveAttachment(t *testing.T) { attachments = append(attachments, attachment) //create a new instance and database object - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -468,7 +495,8 @@ func TestDBDeleteDocument(t *testing.T) { if err == nil { //create a new instance and database object - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -506,7 +534,8 @@ func TestDBDeleteNonExistingDocument(t *testing.T) { if err == nil { //create a new instance and database object - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -645,7 +674,8 @@ func TestRichQuery(t *testing.T) { if err == nil { //create a new instance and database object -------------------------------------------------------- - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} @@ -859,7 +889,8 @@ func TestBatchBatchOperations(t *testing.T) { defer cleanup(database) //create a new instance and database object -------------------------------------------------------- - couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) db := CouchDatabase{CouchInstance: *couchInstance, DBName: database} diff --git a/core/ledger/util/couchdb/couchdbutil_test.go b/core/ledger/util/couchdb/couchdbutil_test.go index 5bf78cf3c5f..be1f4b24e59 100644 --- a/core/ledger/util/couchdb/couchdbutil_test.go +++ b/core/ledger/util/couchdb/couchdbutil_test.go @@ -32,7 +32,8 @@ func TestCreateCouchDBConnectionAndDB(t *testing.T) { cleanup(database) defer cleanup(database) //create a new connection - couchInstance, err := CreateCouchInstance(connectURL, "", "", maxRetries, maxRetriesOnStartup, requestTimeout) + couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, + couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to CreateCouchInstance")) _, err = CreateCouchDatabase(*couchInstance, database) diff --git a/images/couchdb/local.ini b/images/couchdb/local.ini index 9f1ca108f8a..d1f89a4d382 100644 --- a/images/couchdb/local.ini +++ b/images/couchdb/local.ini @@ -7,3 +7,47 @@ [chttpd] bind_address = 0.0.0.0 +[couchdb] +; Specify the location of the database in container. +; Optionally, these directories can be mounted in the host via docker. +database_dir = /opt/couchdb/data/ +view_index_dir = /opt/couchdb/data/ +uri_file = /opt/couchdb/data/couch.uri + +; only allow the admin user to connect +; Uncomment the following statement to enable admin user security. +; default_security = admin_only + +; allow delayed commits since peer manages savepoints and flushing to disk +delayed_commits = true + +[cluster] +; peer maintains a single replica +n = 1 + +; adjust q to set the level of parallelism locally +; recommended to have no more than 10 million documents/shard (q) +; for 100 million documents, q=10 -- at a minimum +q = 8 + +[log] +writer = file +file = /opt/couchdb/logs/couchdb.log +level = info + +; Uncomment the following two statements to enable admin user security. +; [httpd] +; www-authenticate = Basic realm="administrator" + +[couch_httpd_auth] +; Uncomment the following statement to enable admin user security. +; require_valid_user = true +iterations = 1000 ; iterations for password hashing + +; Uncomment the following two statements to enable admin user security. +; [admins] +; admin = admin + +[attachments] +compressible_types = text/*, application/javascript, application/json, application/xml, application/octet-stream +