[FAB-6745] Fix timing issue in sample

When a CA starts, it creates its signing cert and then
starts listening on its listening port.  The fix is to
wait for the server to start listening on the port rather
than waiting for the signing cert file to be created.

See the waitPort function in env.sh, and places where this
is called.  I also had to increase the max time we wait before
failing.

WARNING: This change set is dependent upon the following
fabric-ca change set and should not be merged until it
has been merged:
   https://gerrit.hyperledger.org/r/#/c/15089/

Change-Id: I781e3653bf6846e22f401fe64855fa155ffeb7cb
Signed-off-by: Keith Smith <bksmith@us.ibm.com>

fabric-ca/scripts/env.sh

@@ -50,7 +50,10 @@ CHANNEL_NAME=mychannel
 # Query timeout in seconds
 QUERY_TIMEOUT=15
 
-# Log directory 
+# Setup timeout in seconds (for setup container to complete)
+SETUP_TIMEOUT=120
+
+# Log directory
 LOGDIR=$DATA/logs
 LOGPATH=/$LOGDIR
 
@@ -208,7 +211,7 @@ function initPeerVars {
 # Switch to the current org's admin identity.  Enroll if not previously enrolled.
 function switchToAdminIdentity {
    if [ ! -d $ORG_ADMIN_HOME ]; then
-      dowait "$CA_NAME to start" 10 $CA_LOGFILE $CA_CHAINFILE
+      dowait "$CA_NAME to start" 60 $CA_LOGFILE $CA_CHAINFILE
       log "Enrolling admin '$ADMIN_NAME' with $CA_HOST ..."
       export FABRIC_CA_CLIENT_HOME=$ORG_ADMIN_HOME
       export FABRIC_CA_CLIENT_TLS_CERTFILES=$CA_CHAINFILE
@@ -229,7 +232,7 @@ function switchToUserIdentity {
    export FABRIC_CA_CLIENT_HOME=/etc/hyperledger/fabric/orgs/$ORG/user
    export CORE_PEER_MSPCONFIGPATH=$FABRIC_CA_CLIENT_HOME/msp
    if [ ! -d $FABRIC_CA_CLIENT_HOME ]; then
-      dowait "$CA_NAME to start" 10 $CA_LOGFILE $CA_CHAINFILE
+      dowait "$CA_NAME to start" 60 $CA_LOGFILE $CA_CHAINFILE
       log "Enrolling user for organization $ORG with home directory $FABRIC_CA_CLIENT_HOME ..."
       export FABRIC_CA_CLIENT_TLS_CERTFILES=$CA_CHAINFILE
       fabric-ca-client enroll -d -u https://$USER_NAME:$USER_PASS@$CA_HOST:7054
@@ -251,7 +254,7 @@ function copyAdminCert {
    if $ADMINCERTS; then
       dstDir=$1/admincerts
       mkdir -p $dstDir
-      dowait "$ORG administator to enroll" 10 $SETUP_LOGFILE $ORG_ADMIN_CERT
+      dowait "$ORG administator to enroll" 60 $SETUP_LOGFILE $ORG_ADMIN_CERT
       cp $ORG_ADMIN_CERT $dstDir
    fi
 }
@@ -273,7 +276,7 @@ function finishMSPSetup {
 }
 
 function awaitSetup {
-   dowait "the 'setup' container to finish registering identities, creating the genesis block and other artifacts" $1 $SETUP_LOGFILE /$SETUP_SUCCESS_FILE
+   dowait "the 'setup' container to finish registering identities, creating the genesis block and other artifacts" $SETUP_TIMEOUT $SETUP_LOGFILE /$SETUP_SUCCESS_FILE
 }
 
 # Wait for one or more files to exist
@@ -305,6 +308,36 @@ function dowait {
    echo ""
 }
 
+# Wait for a process to begin to listen on a particular host and port
+# Usage: waitPort <what> <timeoutInSecs> <errorLogFile> <host> <port>
+function waitPort {
+   set +e
+   local what=$1
+   local secs=$2
+   local logFile=$3
+   local host=$4
+   local port=$5
+   nc -z $host $port > /dev/null 2>&1
+   if [ $? -ne 0 ]; then
+      log -n "Waiting for $what ..."
+      local starttime=$(date +%s)
+      while true; do
+         sleep 1
+         nc -z $host $port > /dev/null 2>&1
+         if [ $? -eq 0 ]; then
+            break
+         fi
+         if [ "$(($(date +%s)-starttime))" -gt "$secs" ]; then
+            fatal "Failed waiting for $what; see $logFile"
+         fi
+         echo -n "."
+      done
+      echo ""
+   fi
+   set -e
+}
+
+
 # log a message
 function log {
    if [ "$1" = "-n" ]; then

fabric-ca/scripts/run-fabric.sh

@@ -13,9 +13,9 @@ function main {
 
    done=false
 
-   # Wait for setup to complete and then wait another 5 seconds for the orderer and peers to start
-   awaitSetup 10
-   sleep 5
+   # Wait for setup to complete and then wait another 10 seconds for the orderer and peers to start
+   awaitSetup
+   sleep 10
 
    trap finish EXIT
 

fabric-ca/scripts/setup-fabric.sh

@@ -12,7 +12,6 @@
 #
 
 function main {
-   sleep 1
    log "Beginning building channel artifacts ..."
    registerIdentities
    getCACerts
@@ -22,9 +21,9 @@ function main {
    touch /$SETUP_SUCCESS_FILE
 }
 
-# Enroll as the CA admin
+# Enroll the CA administrator
 function enrollCAAdmin {
-   dowait "$CA_NAME to start" 10 $CA_LOGFILE $CA_CHAINFILE
+   waitPort "$CA_NAME to start" 90 $CA_LOGFILE $CA_HOST 7054
    log "Enrolling with $CA_NAME as bootstrap identity ..."
    export FABRIC_CA_CLIENT_HOME=$HOME/cas/$CA_NAME
    export FABRIC_CA_CLIENT_TLS_CERTFILES=$CA_CHAINFILE

fabric-ca/scripts/start-intermediate-ca.sh

@@ -10,9 +10,8 @@ initOrgVars $ORG
 
 set -e
 
-dowait "Root CA certificate file to be created" 10 $ROOT_CA_CERTFILE $ROOT_CA_LOGFILE
-
-sleep 2
+# Wait for the root CA to start
+waitPort "root CA to start" 60 $ROOT_CA_LOGFILE $ROOT_CA_HOST 7054
 
 # Initialize the intermediate CA
 fabric-ca-server init -b $BOOTSTRAP_USER_PASS -u $PARENT_URL

fabric-ca/scripts/start-orderer.sh

@@ -10,7 +10,7 @@ set -e
 source $(dirname "$0")/env.sh
 
 # Wait for setup to complete sucessfully
-awaitSetup 10
+awaitSetup
 
 # Enroll to get orderer's TLS cert (using the "tls" profile)
 fabric-ca-client enroll -d --enrollment.profile tls -u $ENROLLMENT_URL -M /tmp/tls --csr.hosts $ORDERER_HOST
@@ -30,7 +30,7 @@ finishMSPSetup $ORDERER_GENERAL_LOCALMSPDIR
 copyAdminCert $ORDERER_GENERAL_LOCALMSPDIR
 
 # Wait for the genesis block to be created
-dowait "genesis block to be created" 10 $SETUP_LOGFILE $ORDERER_GENERAL_GENESISFILE
+dowait "genesis block to be created" 60 $SETUP_LOGFILE $ORDERER_GENERAL_GENESISFILE
 
 # Start the orderer
 env | grep ORDERER

fabric-ca/scripts/start-peer.sh

@@ -9,7 +9,7 @@ set -e
 
 source $(dirname "$0")/env.sh
 
-awaitSetup 10
+awaitSetup
 
 # Enroll the peer to get a TLS cert
 fabric-ca-client enroll -d --enrollment.profile tls -u $ENROLLMENT_URL -M /tmp/tls --csr.hosts $PEER_HOST

fabric-ca/start.sh

@@ -46,10 +46,10 @@ log "Creating docker containers ..."
 docker-compose up -d
 
 # Wait for the setup container to complete
-dowait "the 'setup' container to finish registering identities, creating the genesis block and other artifacts" 10 $SDIR/$SETUP_LOGFILE $SDIR/$SETUP_SUCCESS_FILE
+dowait "the 'setup' container to finish registering identities, creating the genesis block and other artifacts" 90 $SDIR/$SETUP_LOGFILE $SDIR/$SETUP_SUCCESS_FILE
 
 # Wait for the run container to start and then tails it's summary log
-dowait "the docker 'run' container to start" 15 ${SDIR}/${SETUP_LOGFILE} ${SDIR}/${RUN_SUMFILE}
+dowait "the docker 'run' container to start" 60 ${SDIR}/${SETUP_LOGFILE} ${SDIR}/${RUN_SUMFILE}
 tail -f ${SDIR}/${RUN_SUMFILE}&
 TAIL_PID=$!