[FAB-5782] Initialization failure on Postgres

PostgreSQL always expects to connect to a database, it does not
allow establishing a connection just to the server itself. This
causes issues when you specify a database in the connection string
that does not exist.

A default PostgresSQL installation comes with two default database
that can be used to establish a connection - "template1" and "postgres".
Connection retry logic is implemented  as defined below:

1. Connect to database specified in connection string
2. If 1 fails, connect to "postgres" database
3. If 2 fails, connect to "template1" database
4. If 1-3 fail, throw an error indicating database initialization failed
and that database needs to be created manually. The server will continue to
start but will try to reinitialize the database on the next request the server
receives.

Change-Id: I84f3d52a2b770ec1002fe8715f31cd1dfc9c2e5b
Signed-off-by: Saad Karim <skarim@us.ibm.com>

lib/dbutil/dbutil.go

@@ -88,7 +88,7 @@ func createSQLiteDBTables(datasource string) error {
 	return nil
 }
 
-// NewUserRegistryPostgres opens a connecton to a postgres database
+// NewUserRegistryPostgres opens a connection to a postgres database
 func NewUserRegistryPostgres(datasource string, clientTLSConfig *tls.ClientTLSConfig) (*sqlx.DB, error) {
 	log.Debugf("Using postgres database, connecting to database...")
 
@@ -110,22 +110,33 @@ func NewUserRegistryPostgres(datasource string, clientTLSConfig *tls.ClientTLSCo
 		datasource = fmt.Sprintf("%s sslcert=%s sslkey=%s", datasource, cert, key)
 	}
 
-	connStr := getConnStr(datasource)
+	dbNames := []string{dbName, "postgres", "template1"}
+	var db *sqlx.DB
+	var pingErr, err error
 
-	log.Debug("Connecting to PostgreSQL server, using connection string: ", MaskDBCred(connStr))
-	db, err := sqlx.Open("postgres", connStr)
-	if err != nil {
-		return nil, errors.Wrap(err, "Failed to open Postgres database")
+	for _, dbName := range dbNames {
+		connStr := getConnStr(datasource, dbName)
+		log.Debug("Connecting to PostgreSQL server, using connection string: ", MaskDBCred(connStr))
+
+		db, err = sqlx.Open("postgres", connStr)
+		if err != nil {
+			return nil, errors.Wrap(err, "Failed to open Postgres database")
+		}
+
+		pingErr = db.Ping()
+		if pingErr == nil {
+			break
+		}
+		log.Warningf("Failed to connect to database '%s'", dbName)
 	}
 
-	err = db.Ping()
-	if err != nil {
-		return nil, errors.Wrap(err, "Failed to connect to Postgres database")
+	if pingErr != nil {
+		return nil, errors.Errorf("Failed to connect to Postgres database. Postgres requires connecting to a specific database, the following databases were tried: %s. Please create one of these database before continuing", dbNames)
 	}
 
 	err = createPostgresDatabase(dbName, db)
 	if err != nil {
-		return nil, errors.Wrap(err, "Failed to create Postgres database: %s")
+		return nil, errors.Wrap(err, "Failed to create Postgres database")
 	}
 
 	log.Debugf("Connecting to database '%s', using connection string: '%s'", dbName, MaskDBCred(datasource))
@@ -277,9 +288,9 @@ func getDBName(datasource string) string {
 }
 
 // GetConnStr gets connection string without database
-func getConnStr(datasource string) string {
+func getConnStr(datasource string, dbname string) string {
 	re := regexp.MustCompile(`(dbname=)([^\s]+)`)
-	connStr := re.ReplaceAllString(datasource, "")
+	connStr := re.ReplaceAllString(datasource, fmt.Sprintf("dbname=%s", dbname))
 	return connStr
 }
 

scripts/fvt/postgres_test.sh

@@ -0,0 +1,140 @@
+#!/bin/bash
+#
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+TESTCASE="postgres"
+FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca"
+FABRIC_CAEXEC="$FABRIC_CA/bin/fabric-ca"
+SCRIPTDIR="$FABRIC_CA/scripts/fvt"
+. $SCRIPTDIR/fabric-ca_utils
+RC=0
+
+export FABRIC_CA_SERVER_HOME="/tmp/$TESTCASE"
+
+PGSQLSERVERCONFIG="$FABRIC_CA_SERVER_HOME/pgsqlserverconfig.yaml"
+SERVERLOG="$FABRIC_CA_SERVER_HOME/serverlog.txt"
+MSP="$FABRIC_CA_SERVER_HOME/msp"
+SERVERCERT="$FABRIC_CA_SERVER_HOME/fabric-ca-cert.pem"
+DBNAME="fabric_ca"
+
+function cleanup {
+    rm $SERVERCERT
+    rm -rf $MSP
+    rm $SERVERLOG
+}
+
+function configureDB {
+    psql -c "CREATE USER testuser WITH PASSWORD 'testuserpw' LOGIN"
+    psql -c "CREATE DATABASE testdb"
+    psql -d testdb -c "DROP DATABASE $DBNAME"
+    psql -d testdb -c "DROP DATABASE postgres"
+}
+
+function resetDB {
+    psql -d testdb -c "ALTER DATABASE template1_temp RENAME TO template1"
+    psql -d testdb -c "CREATE DATABASE $DBNAME"
+    psql -d testdb -c "CREATE DATABASE postgres"
+    psql -d testdb -c "ALTER USER testuser WITH NOCREATEDB"
+}
+
+function genConfig {
+    postgresTls='sslmode=disable'
+   case "$FABRIC_TLS" in
+      true) postgresTls='sslmode=require' ;;
+   esac
+
+   mkdir -p $FABRIC_CA_SERVER_HOME
+   cat > $PGSQLSERVERCONFIG <<EOF
+debug: true
+
+db:
+  type: postgres
+  datasource: host=localhost port=$POSTGRES_PORT user=testuser password=testuserpw dbname=fabric_ca $postgresTls
+  tls:
+     enabled: $FABRIC_TLS
+     certfiles:
+       - $TLS_ROOTCERT
+     client:
+       certfile: $TLS_CLIENTCERT
+       keyfile: $TLS_CLIENTKEY
+
+tls:
+  enabled: $FABRIC_TLS
+  certfile: $TLS_SERVERCERT
+  keyfile: $TLS_SERVERKEY
+
+registry:
+  # Maximum number of times a password/secret can be reused for enrollment
+  # (default: -1, which means there is no limit)
+  maxenrollments: -1
+
+  # Contains identity information which is used when LDAP is disabled
+  identities:
+     - name: a
+       pass: b
+       type: client
+       affiliation: ""
+       maxenrollments: -1
+       attrs:
+          hf.Registrar.Roles: "client,user,peer,validator,auditor"
+          hf.Registrar.DelegateRoles: "client,user,validator,auditor"
+          hf.Revoker: true
+          hf.IntermediateCA: true
+
+affiliations:
+   org1:
+      - department1
+      - department2
+   org2:
+      - department1
+EOF
+}
+
+genConfig
+cleanup
+configureDB
+
+# TEST 1: Database user does not have permission to create DB and also
+# no database exists with the same name as user
+$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG 2>&1 | tee $SERVERLOG &
+pollServer fabric-ca-server 127.0.0.1 17054 10 start
+$SCRIPTDIR/fabric-ca_setup.sh -K
+grep "pq: permission denied to create database" $SERVERLOG &> /dev/null
+if [ $? != 0 ]; then
+    ErrorMsg "'testuser' should not have been able to create database, does not have permissions"
+fi
+
+# TEST 2: There are no database to establish a connection, an error is expected
+# Three database are tried, the database specified in connection string, postgres,
+# and template1
+psql -d testdb -c "ALTER DATABASE template1 RENAME TO template1_temp"
+$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG 2>&1 | tee $SERVERLOG &
+pollServer fabric-ca-server 127.0.0.1 17054 10 start
+grep "Please create one of these database before continuing" $SERVERLOG &> /dev/null
+if [ $? != 0 ]; then
+    ErrorMsg "None of the database expected exist, should have thrown an error in the logs"
+fi
+
+# TEST 3: User has permissions to create DB and at least of the expected database
+# exists, should successfully initialize database now
+psql -d testdb -c "ALTER DATABASE template1_temp RENAME TO template1"
+psql -d testdb -c "ALTER USER testuser WITH CREATEDB"
+
+# Enroll should try to reinitialize the DB before processing enroll request and should succeed
+enroll a b 2>&1 | grep "Stored client certificate"
+if [ $? != 0 ]; then
+    ErrorMsg "Enroll request should have passed"
+fi
+
+$SCRIPTDIR/fabric-ca_setup.sh -K
+grep "Initialized postgres database" $SERVERLOG &> /dev/null
+if [ $? != 0 ]; then
+    ErrorMsg "Postgres database should have been successfully initialized"
+fi
+
+resetDB
+CleanUp $RC
+exit $RC