From ab7a40b2a032ae8afddac0b62684c59ad544454e Mon Sep 17 00:00:00 2001 From: Allen Bailey Date: Thu, 19 Oct 2017 16:51:48 -0400 Subject: [PATCH] [ FAB-6698 ] Fabric CA clustering Add a test that includes multi-ca capability in conjunction with intermediate CAs in a cluster, with database sharing, with a proxy front-end. Test test will accept arguments that will allow scaling. It will build a large command list to be run concurrently. The defaults are Number of Root CA instances: 2 Number of Intermediate CA instances: 2 Number of multicas per instance: 2 Number of interations: 16 Largest Number of concurrent interactions: 96 Some changes were required for the tests to run successfully: 1 -There were some tests using shared DBs that needed to be updated, since the expected default for multicas is now *non*-shared. 2 - A change to the setup script that now allows arbitrary leftover command-line arguments to be passed to the fabric-ca-server command exposed a bug in scripts/fvt/cdp_exploit_test.sh that was introduced in https://gerrit.hyperledger.org/r/#/c/11571/ -- "(Improve error handling (#3)" -- which was also fixed. 3 - The haproxy timeouts were insufficient for large numbers of connections. Increased to 5 minutes: 4 - Lastly, end-of-line spaces were removed from a few random files. Change-Id: Ic28c7beaffc3f400b6bd7fefbb76363332830dc2 Signed-off-by: Allen Bailey --- images/fabric-ca-fvt/payload/mysql_setup.sh | 1 + .../fabric-ca-fvt/payload/postgres_setup.sh | 2 + images/fabric-ca-fvt/payload/system_update.sh | 2 +- scripts/fvt/cdp_exploit_test.sh | 4 +- scripts/fvt/cluster_test.sh | 397 ++++++++++++++++++ scripts/fvt/fabric-ca_setup.sh | 192 ++++++--- scripts/fvt/fabric-ca_utils | 44 +- scripts/fvt/intermediateca_test.sh | 2 +- scripts/fvt/multica_test.sh | 2 +- 9 files changed, 566 insertions(+), 80 deletions(-) create mode 100755 scripts/fvt/cluster_test.sh diff --git a/images/fabric-ca-fvt/payload/mysql_setup.sh b/images/fabric-ca-fvt/payload/mysql_setup.sh index 9ca329456..07c6461ca 100755 --- a/images/fabric-ca-fvt/payload/mysql_setup.sh +++ b/images/fabric-ca-fvt/payload/mysql_setup.sh @@ -13,6 +13,7 @@ chown mysql.mysql $MYSQLDATA/*pem chmod 600 $MYSQLDATA/$TLS_SERVER_KEY test $arch = s390x && MYCNF=/etc/mysql/my.cnf || MYCNF=/etc/mysql/mysql.conf.d/mysqld.cnf sed -i "s/^[[:blank:]]*#*[[:blank:]]*ssl-ca=.*/ssl-ca=$TLS_BUNDLE/; + s/\(^[[:blank:]]*\)#*\([[:blank:]]*max_connections[[:blank:]]*=[[:blank:]]*\).*/\1\22000/; s/^[[:blank:]]*#*[[:blank:]]*ssl-cert=.*/ssl-cert=$TLS_SERVER_CERT/; s/^[[:blank:]]*#*[[:blank:]]*ssl-key=.*/ssl-key=$TLS_SERVER_KEY/" $MYCNF || let RC+=1 chown -R mysql.mysql $MYSQLDATA diff --git a/images/fabric-ca-fvt/payload/postgres_setup.sh b/images/fabric-ca-fvt/payload/postgres_setup.sh index 71062a9ef..0d9c3c7bf 100755 --- a/images/fabric-ca-fvt/payload/postgres_setup.sh +++ b/images/fabric-ca-fvt/payload/postgres_setup.sh @@ -20,6 +20,8 @@ cp $FABRIC_CA_DATA/$TLS_SERVER_KEY $PGDATA || let RC+=1 chown $PGUSER.$PGUSER $PGDATA/*pem || let RC+=1 chmod 600 $PGDATA/FabricTlsServer*.pem || let RC+=1 sed -i "s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl[[:blank:]]*=[[:blank:]]*\).*/\1\2on/;\ +s/\(^[[:blank:]]*\)#*\([[:blank:]]*max_connections[[:blank:]]*=[[:blank:]]*\).*/\1\22000/;\ +s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl_cert_file[[:blank:]]*=[[:blank:]]*\).*/\1\2'$TLS_SERVER_CERT'/;\ s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl_cert_file[[:blank:]]*=[[:blank:]]*\).*/\1\2'$TLS_SERVER_CERT'/;\ s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl_key_file[[:blank:]]*=[[:blank:]]*\).*/\1\2'$TLS_SERVER_KEY'/;\ s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl_ca_file[[:blank:]]*=[[:blank:]]*\).*/\1\2'$TLS_BUNDLE'/;\ diff --git a/images/fabric-ca-fvt/payload/system_update.sh b/images/fabric-ca-fvt/payload/system_update.sh index b808e2c08..8dabc8501 100755 --- a/images/fabric-ca-fvt/payload/system_update.sh +++ b/images/fabric-ca-fvt/payload/system_update.sh @@ -22,7 +22,7 @@ echo "mysql-server mysql-server/root_password_again password mysql" | debconf-se apt-get -y install --no-install-recommends rsyslog bc vim lsof sqlite3 haproxy postgresql-$PGVER \ postgresql-client-common postgresql-contrib-$PGVER isag jq git html2text \ debconf-utils zsh htop python2.7-minimal libpython2.7-stdlib \ - mysql-client mysql-common mysql-server || let RC+=1 + mysql-client mysql-common mysql-server parallel || let RC+=1 apt-get -y install ssl-cert || let RC+=1 apt-get -y autoremove diff --git a/scripts/fvt/cdp_exploit_test.sh b/scripts/fvt/cdp_exploit_test.sh index cd3177d87..391015629 100755 --- a/scripts/fvt/cdp_exploit_test.sh +++ b/scripts/fvt/cdp_exploit_test.sh @@ -1,4 +1,4 @@ - !/bin/bash +# !/bin/bash # # Copyright IBM Corp. All Rights Reserved. # @@ -71,7 +71,7 @@ function testLimit { export FABRIC_CA_SERVER_CRLSIZELIMIT=$1 cd $GOPATH/src/github.com/hyperledger/fabric-ca $SCRIPTDIR/fabric-ca_setup.sh -K - $SCRIPTDIR/fabric-ca_setup.sh -S -D -X $ca_certfile -k $ca_keyfile 2>&1 | tee $LOGFILE & + $SCRIPTDIR/fabric-ca_setup.sh -S -D -X -c $ca_certfile -k $ca_keyfile 2>&1 | tee $LOGFILE & pollLogForMsg "Listening on https*://0.0.0.0:$CA_DEFAULT_PORT" $LOGFILE || ErrorExit "Failed to log CA" enroll admin2 adminpw2 admin_keyfile="$(find /tmp/$TESTCASE/admin/msp/keystore -type f)" diff --git a/scripts/fvt/cluster_test.sh b/scripts/fvt/cluster_test.sh new file mode 100755 index 000000000..6bee1021e --- /dev/null +++ b/scripts/fvt/cluster_test.sh @@ -0,0 +1,397 @@ +#!/bin/bash + +: ${TESTCASE="ca_cluster"} +FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca" +SCRIPTDIR="$FABRIC_CA/scripts/fvt" +. $SCRIPTDIR/fabric-ca_utils +ROOTDIR=/tmp/cluster +INTDIR=$ROOTDIR/int +ROOTUSERDIR=$ROOTDIR/users +INTUSERDIR=$ROOTDIR/int/users +ENROLLCERT=msp/signcerts/cert.pem +DEFAULT_CA_CONFIG=fabric-ca-config.yaml +RC=0 +DBNAME=fabric_ca +INTDBNAME=intfabric_ca +NUMSERVER="$1" # Number of CA instances behind the proxy +NUMINTERMD="$2" # Number of intermediate CAs +NUMCAS="$3" # cacount; as a simplifying assumption, + # if NUMSERVER > NUMCAS, + # then NUMSERVER % NUMCAS = 0 + # else NUMCAS % NUMSERVER = 0 +ITERATIONS="$4" # num of commands to run in parallel + # As a simplifying assumption, ITERATIONS % 4 = 0 +NUMJOBS="$5" # num of concurrent jobs + +# Using the default, the test should complete in ≈ 1 min +: ${NUMSERVER:=2} +: ${NUMINTERMD:=2} +: ${NUMCAS:=2} +: ${ITERATIONS:=8} +: ${NUMJOBS:=1024} # spawn as many jobs as there are potential requests +NUMUSERS=$((NUMCAS*ITERATIONS)) # NUMUSERS % NUMSERVER should = 0 +USERNAME="testuser" +ROOT_CA_ADDR=localhost +INTUSER="intermediateCa16" +INTPSWD="intermediateCa16pw" +SHELL=/bin/bash + +setTLS +export SHELL PROXY_PORT ROOTDIR ITERATIONS USERNAME PROTO TLSOPT FABRIC_CA_CLIENTEXEC ENROLLCERT + +function enrollAdmin() { + local port="$1" + local ca="$2" + local dir="$3" + : ${port:=$PROXY_PORT} + + mkdir -p $dir/admin$ca + touch $dir/admin$ca/log.txt + FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ + $FABRIC_CA_CLIENTEXEC enroll --debug $TLSOPT \ + -u ${PROTO}admin:adminpw@localhost:$port \ + --csr.hosts admin@fab-client.raleigh.ibm.com \ + --csr.hosts admin.fabric.raleigh.ibm.com,127.0.0.2 \ + --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 +} + +function revokeUsers() { + local port="$1" + local ca="$2" + local user="$3" + local dir="$4" + : ${port:=$PROXY_PORT} + + FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ + $FABRIC_CA_CLIENTEXEC revoke --debug $TLSOPT \ + -u ${PROTO}admin:adminpw@localhost:$port \ + --revoke.name ${USERNAME}${ca}-${user} \ + --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 +} + +function enrollUsers() { + local port="$1" + local ca="$2" + local user="$3" + local dir="$4" + : ${port:=$PROXY_PORT} + mkdir -p $dir/admin$ca + touch $dir/admin$ca/log.txt + FABRIC_CA_CLIENT_HOME=$dir/${USERNAME}${ca}-${user} \ + $FABRIC_CA_CLIENTEXEC enroll --debug $TLSOPT \ + -u ${PROTO}${USERNAME}${ca}-${user}:${USERNAME}${ca}-${user}@localhost:$port \ + --csr.hosts ${USERNAME}${ca}-${user}@fab-client.raleigh.ibm.com \ + --csr.hosts ${USERNAME}${ca}-${user}.fabric.raleigh.ibm.com \ + --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 + test "${USERNAME}${ca}-${user}" = "$(openssl x509 -in $dir/${USERNAME}${ca}-${user}/$ENROLLCERT -noout -subject | awk -F'=' '{print $NF}')" +} + +function reenrollUsers() { + local port="$1" + local ca="$2" + local user="$3" + local dir="$4" + : ${port:=$PROXY_PORT} + mkdir -p $dir/admin$ca + touch $dir/admin$ca/log.txt + FABRIC_CA_CLIENT_HOME=$dir/${USERNAME}${ca}-${user} \ + $FABRIC_CA_CLIENTEXEC reenroll --debug $TLSOPT \ + -u ${PROTO}@localhost:$port \ + --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 + test "${USERNAME}${ca}-${user}" = "$(openssl x509 -in $dir/${USERNAME}${ca}-${user}/$ENROLLCERT -noout -subject | awk -F'=' '{print $NF}')" +} + +function register() { + local port="$1" + local ca="$2" + local user="$3" + local dir="$4" + : ${port:=$PROXY_PORT} + FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ + $FABRIC_CA_CLIENTEXEC register --debug -u ${PROTO}localhost:$port $TLSOPT \ + --id.name ${USERNAME}${ca}-${user} \ + --id.secret ${USERNAME}${ca}-${user} \ + --id.type client \ + --id.maxenrollments $ITERATIONS \ + --id.affiliation bank_a \ + --id.attrs test=testValue \ + --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 +} + +function DBvalidateUsers() { + # Query the DB and verify the user state: + # 0 - registered, but not enrolled + # 1 - enrolled + local state="$1" + local dbname="$2" + local StatusField=6 + local fsopt="" + + case $DBdriver in + postgres) StatusField=11 ;; + esac + + DBNAME=$dbname $SCRIPTDIR/fabric-ca_setup.sh -L -d $DBdriver \ + -n $NUMSERVER -u $NUMCAS 2>/dev/null | + awk -v u="$USERNAME" $fsopt \ + -v s="$state" \ + -v n="$NUMUSERS" \ + -v f="$StatusField" \ + -v t=0 ' + $1~u && $f==s {t++} + END { if (t!=n) exit 1 }' +} + +function showUsers() { + $SCRIPTDIR/fabric-ca_setup.sh -L -d $DBdriver \ + -n $NUMSERVER -u $NUMCAS 2>/dev/null | + awk -v u="$USERNAME" '$1~u' +} + +function verifyDistribution() { + case "$FABRIC_TLS" in + 1|y|yes|true) return 0 ;; + esac + + local numCluster="$1" + local backendName="$2" + local expectedCount="$3" + local percentage="$4" + + for s in $(seq $numCluster); do + verifyServerTraffic "" "/$backendName$((s-1))""" $expectedCount $percentage + test $? -eq 0 && + echo " >>>>>>>>>> VerifyServerTraffic for $backendName$s...PASSED" || + ErrorMsg " >>>>>>>>>> VerifyServerTraffic for $backendName$s...FAILED" + done +} + +export -f enrollAdmin register enrollUsers revokeUsers reenrollUsers + +function checkStatus() { + # Parse the joblog exitstatus (column 7) for all jobs + # 0 - success + # ^0 - failed + # Success is measured by the number of successful jobs, i.e., + # there should be one successful job for each request sent: + # Number of exit '0' entries == NUMUSERS + local log="$1" + local number="$2" + : ${number:="$NUMUSERS"} + awk -v u=$number ' + NR!=1 && $7==0 {rc+=1} + END {if (rc!=u) exit 1}' $log + test $? -ne 0 && ErrorMsg "FAILED" || echo "PASSED" +} + + +for DBdriver in mysql postgres; do + echo "Testing $DBdriver >>>>>>>>>>>>>" + # Delete all of the DBs + echo -e " >>>>>>>>>> Deleting all databases" + $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS + DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS + rm -rf $ROOTDIR + + # Initialize all of the configs, certs, keys and directories + mkdir -p $ROOTUSERDIR + mkdir -p $INTUSERDIR + echo -e " >>>>>>>>>> Initializing Root CAs" + $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -I -n 1 -u $NUMCAS \ + -n $NUMSERVER -D -d $DBdriver > $ROOTDIR/log.txt 2>&1 + echo -e " >>>>>>>>>> Initializing Intermediate CAs" + DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -D -d $DBdriver -u $NUMCAS -I -r $INTERMEDIATE_CA_DEFAULT_PORT -x $INTDIR \ + -U "${PROTO}$INTUSER:$INTPSWD@$ROOT_CA_ADDR:$CA_DEFAULT_PORT" > $INTDIR/log.txt 2>&1 + + ################################################################## + ## Customize enrollment for each CA + ################################################################## + ca=0 + rootCafiles="" + intermediateCafiles="" + rootDBconfig="" + intermediateDBconfig="" + # append the customized DB config to each CA's config file + while test $((ca++)) -lt $NUMCAS; do + # build the list of cafiles to be passed to server start + rootCafiles="$rootCafiles,$ROOTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG}" + intermediateCafiles="$intermediateCafiles,$INTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG}" + + # each intermediate CA needs an enrollment identity and parentserver.url + # each also needs a unique caname, or the sever start will fail + enrollment=" +intermediate: + parentserver: + url: ${PROTO}intermediateCa${ca}:intermediateCa${ca}pw@127.0.0.1:$CA_DEFAULT_PORT + caname: ca${ca} + enrollment: + name: intermediateCa${ca} + secret: intermediateCa${ca}pw + hosts: + - localhost +" + # append the intermediate CA config to each CA's config + cat >> $INTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG} <>>>>>>>>> Starting $NUMSERVER Root CA instances with $NUMCAS servers each" + $SCRIPTDIR/fabric-ca_setup.sh -N -X -o 30 -x $ROOTDIR -S -n $NUMSERVER -D -d $DBdriver \ + -- "--cafiles" "$rootCafiles" >> $ROOTDIR/log.txt 2>&1 || + ErrorExit "Failed to start root servers" + echo -e " >>>>>>>>>> Starting $NUMSERVER Intermediate CA instances with $NUMCAS servers each" + $SCRIPTDIR/fabric-ca_setup.sh -o 30 -n $NUMSERVER -S -r $INTERMEDIATE_CA_DEFAULT_PORT -x $INTDIR \ + -U "https://$INTUSER:$INTPSWD@$ROOT_CA_ADDR:$PROXY_PORT" \ + -- "--cafiles" "$intermediateCafiles" >> $INTDIR/log.txt 2>&1 || + ErrorExit "Failed to intermediate servers" + + ######################################################### + # The bulk of the work comes here -- + # register and enroll users, in parallel, $NUMJOBS at a time + ######################################################### + for SERVER in $PROXY_PORT $INTERMEDIATE_PROXY_PORT ; do + # The intermediate CAs do not share the root CA's DBs; + # each has a unique DB (in a unique directory, if file-based + # If the CA is root, the haproxy backend name is 'server' + # If the CA is intermediate, the haproxy backend name is 'intserver' + if test "$SERVER" = "$INTERMEDIATE_PROXY_PORT"; then + dbname=$INTDBNAME + userdir=$INTUSERDIR + stype=root + backend=intserver + else + dbname=$DBNAME + userdir=$ROOTUSERDIR + stype=intermediate + backend=server + fi + + count=0 + # for each block of requests, we will validate round-robin + # distribution to all participating CAs + verifyDistribution $NUMSERVER $backend $count 0 + + echo -e " >>>>>>>>>> Testing $stype CA using DB name $dbname" + + # Enroll the admins -- the total number of enrollment requests + # sent is calulated to be the larger of NUMSERVER | NUMCAS + test $NUMCAS -ge $NUMSERVER && numReq=1 || numReq=$((NUMSERVER/NUMCAS)) + printf " >>>>>>>>>> Enrolling ${NUMCAS} admins, $numReq times..." + parallel -k --no-notice --jobs $NUMJOBS --joblog $userdir/adminEnroll.log \ + enrollAdmin $SERVER {1} $userdir ::: $(seq $NUMCAS) ::: $(seq $numReq) + checkStatus $userdir/adminEnroll.log $((numReq*NUMCAS)) || ErrorExit "Enroll of admins failed" + # Register $NUMUSERS users and validate their status in the DB + test $NUMCAS -lt $NUMSERVER && count=1 || count=$((NUMCAS/NUMSERVER)) + verifyDistribution $NUMSERVER $backend $count + + # Register $NUMUSERS users and validate their status in the DB + printf " >>>>>>>>>> Registering $NUMUSERS users (${NUMCAS}x${ITERATIONS})..." + parallel --no-notice --jobs $NUMJOBS --joblog $userdir/register.log \ + register $SERVER {1} {2} $userdir ::: $(seq $NUMCAS) ::: $(seq $ITERATIONS) + checkStatus $userdir/register.log + DBvalidateUsers 0 $dbname && + echo -e " >>>>>>>>>> Validating user status in DB...PASSED" || + ErrorMsg " >>>>>>>>>> Validating user status in DB...FAILED" + count=$((count+$((ITERATIONS*NUMCAS/NUMSERVER)) )) + verifyDistribution $NUMSERVER $backend $count + + # Enroll $NUMUSERS users and validate their status in the DB + printf " >>>>>>>>>> Enrolling $NUMUSERS users (${NUMCAS}x${ITERATIONS})..." + parallel --no-notice --jobs $NUMJOBS --joblog $userdir/userEnroll.log \ + enrollUsers $SERVER {1} {2} $userdir ::: $(seq $NUMCAS) ::: $(seq $ITERATIONS) + checkStatus $userdir/userEnroll.log + DBvalidateUsers 1 $dbname && + echo -e " >>>>>>>>>> Validating user status in DB...PASSED" || + ErrorMsg " >>>>>>>>>> Validating user status in DB...FAILED" + count=$((count+$((ITERATIONS*NUMCAS/NUMSERVER)) )) + verifyDistribution $NUMSERVER $backend $count + + # Do all of the following simultaneously + # enroll Enroll an identity + # getcacert Get CA certificate chain + # reenroll Reenroll an identity + # register Register an identity + # revoke Revoke an identity + # gencrl Generate a CRL + + > /tmp/cmd.lst + for ca in $(seq $NUMCAS); do + # Create the register cmd list of brand new users where + # the previous register task left off + echo " >>>>>>> generating register command list ($((ITERATIONS*NUMCAS)))" + for user in $(seq $((ITERATIONS+1)) $((ITERATIONS+ITERATIONS)) ); do + echo register $SERVER $ca $user $userdir >> /tmp/cmd.lst + done + # Create the enroll cmd list - + # issue enroll for the first 1/2 of the previously enrolled users + echo " >>>>>>> generating enroll command list ($((ITERATIONS/2*NUMCAS)))" + for user in $(seq $((ITERATIONS/2)) ); do + echo enrollUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst + done + # Create the renroll cmd list - + # issue renroll for the third 1/4 of the previously enrolled users + echo " >>>>>>> generating renroll command list ($((ITERATIONS/4*NUMCAS)))" + for user in $(seq $((ITERATIONS/2+1)) $((ITERATIONS/4*3)) ); do + echo reenrollUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst + done + # Create the revoke cmd list - + # issue renroll for the last 1/4 of the previously enrolled users + echo " >>>>>>> generating revoke command list ($((ITERATIONS/4*NUMCAS)))" + for user in $(seq $((ITERATIONS/4*3+1)) $ITERATIONS ); do + echo revokeUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst + done + # Create the getcacert cmd list - + echo " >>>>>>> generating getcacert command list ($((ITERATIONS*NUMCAS/2)))" + for user in $(seq $((ITERATIONS/2)) ); do + echo "FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC getcacert --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/cacert.txt 2>&1" >> /tmp/cmd.lst + done + # Create the gencrl cmd list - + echo " >>>>>>> generating gencrl command list ($((ITERATIONS*NUMCAS/2)))" + for user in $(seq $((ITERATIONS/2+1)) $ITERATIONS); do + echo "FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC gencrl --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/crl.txt 2>&1" >> /tmp/cmd.lst + done + done + + shuf --output=$userdir/cmd.lst /tmp/cmd.lst + # OK, here goes... + printf " >>>>>>>>>> Executing all $((ITERATIONS*3*NUMCAS)) jobs..." + parallel --no-notice --jobs $NUMJOBS --joblog $userdir/cmd.log < $userdir/cmd.lst + checkStatus $userdir/cmd.log $((ITERATIONS*3*NUMCAS)) + count=$((count+$((ITERATIONS*3*NUMCAS/NUMSERVER)))) + sleep 1 + verifyDistribution $NUMSERVER $backend $count + + sleep 1 + # Lastly, 1/4 of user certs should be revoked + echo " >>>>>>>>>> Checking crl; expect $((ITERATIONS/4)) revoked certificates..." + for ca in $(seq $NUMCAS); do + FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC gencrl --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/crl.txt 2>&1 + revoked=$(openssl crl -in $userdir/admin$ca/msp/crls/crl.pem -text -noout | grep -c 'Serial Number:') + test $revoked -eq $((ITERATIONS/4)) && + echo -e " >>>>>>>>>> crl check for ca$ca ...PASSED" || + ErrorMsg " >>>>>>>>>> crl check for ca$ca...FAILED got ('$revoked') revoked certs on localhost:$SERVER" + done + done + echo "" + echo "" +done + +# Delete all of the DBs +echo -e " >>>>>>>>>> Deleting all databases" +$SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS +DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS +CleanUp $RC +exit $RC diff --git a/scripts/fvt/fabric-ca_setup.sh b/scripts/fvt/fabric-ca_setup.sh index f63e4d299..ec3fb42e0 100755 --- a/scripts/fvt/fabric-ca_setup.sh +++ b/scripts/fvt/fabric-ca_setup.sh @@ -49,11 +49,16 @@ runPSQL() { resetFabricCa(){ killAllFabricCas rm -rf $DATADIR >/dev/null - test -f $(pwd)/$DBNAME && rm $(pwd)/$DBNAME + test -f $(pwd)/${DBNAME}* && rm $(pwd)/${DBNAME}* cd /tmp - mysql --host=localhost --user=root --password=mysql -e 'show tables' $DBNAME 2>/dev/null && - mysql --host=localhost --user=root --password=mysql -e "DROP DATABASE IF EXISTS $DBNAME" - /usr/bin/dropdb "$DBNAME" -U postgres -h localhost -w --if-exists 2>/dev/null + + # Base server and cluster servers + for i in "" $(seq ${CACOUNT:-0}); do + test -z $i && dbSuffix="" || dbSuffix="_ca$i" + mysql --host=localhost --user=root --password=mysql -e 'show tables' ${DBNAME}${dbSuffix} >/dev/null 2>&1 + mysql --host=localhost --user=root --password=mysql -e "DROP DATABASE IF EXISTS ${DBNAME}${dbSuffix}" >/dev/null 2>&1 + /usr/bin/dropdb "${DBNAME}${dbSuffix}" -U postgres -h localhost -w --if-exists 2>/dev/null + done } listFabricCa(){ @@ -64,50 +69,58 @@ listFabricCa(){ lsof -n -i tcp:$((port+$inst)) inst=$((inst+1)) done - case $DRIVER in - mysql) - echo "" - mysql --host=localhost --user=root --password=mysql -e 'SELECT * FROM users;' $DBNAME - echo "Users:" - mysql --host=localhost --user=root --password=mysql -e 'SELECT * FROM users;' $DBNAME - if $($FABRIC_CA_DEBUG); then - echo "Certificates:" - mysql --host=localhost --user=root --password=mysql -e 'SELECT * FROM certificates;' $DBNAME - echo "Affiliations:" - mysql --host=localhost --user=root --password=mysql -e 'SELECT * FROM affiliations;' $DBNAME - fi - ;; - postgres) - echo "" - runPSQL "\l $DBNAME" | sed 's/^/ /;1s/^ *//;1s/$/:/' - - echo "Users:" - runPSQL "SELECT * FROM USERS;" "--dbname=$DBNAME" | sed 's/^/ /' - if $($FABRIC_CA_DEBUG); then - echo "Certificates::" - runPSQL "SELECT * FROM CERTIFICATES;" "--dbname=$DBNAME" | sed 's/^/ /' - echo "Affiliations:" - runPSQL "SELECT * FROM AFFILIATIONS;" "--dbname=$DBNAME" | sed 's/^/ /' - fi - ;; - sqlite3) sqlite3 "$DATASRC" 'SELECT * FROM USERS ;;' | sed 's/^/ /' - if $($FABRIC_CA_DEBUG); then - sqlite3 "$DATASRC" 'SELECT * FROM CERTIFICATES;' | sed 's/^/ /' - sqlite3 "$DATASRC" 'SELECT * FROM AFFILIATIONS;' | sed 's/^/ /' - fi - esac + + # Base server and cluster servers + for i in "" $(seq ${CACOUNT:-0}); do + test -z $i && dbSuffix="" || dbSuffix="_ca$i" + echo "" + echo " ======================================" + echo " ========> Dumping ${DBNAME}${dbSuffix} Database" + echo " ======================================" + case $DRIVER in + mysql) + echo "" + echo "Users:" + mysql --host=localhost --user=root --password=mysql -e 'SELECT * FROM users;' ${DBNAME}${dbSuffix} + if $($FABRIC_CA_DEBUG); then + echo "Certificates:" + mysql --host=localhost --user=root --password=mysql -e 'SELECT * FROM certificates;' ${DBNAME}${dbSuffix} + echo "Affiliations:" + mysql --host=localhost --user=root --password=mysql -e 'SELECT * FROM affiliations;' ${DBNAME}${dbSuffix} + fi + ;; + postgres) + echo "" + runPSQL "\l ${DBNAME}${dbSuffix}" | sed 's/^/ /;1s/^ *//;1s/$/:/' + + echo "Users:" + runPSQL "SELECT * FROM USERS;" "--dbname=${DBNAME}${dbSuffix}" | sed 's/^/ /' + if $($FABRIC_CA_DEBUG); then + echo "Certificates::" + runPSQL "SELECT * FROM CERTIFICATES;" "--dbname=${DBNAME}${dbSuffix}" | sed 's/^/ /' + echo "Affiliations:" + runPSQL "SELECT * FROM AFFILIATIONS;" "--dbname=${DBNAME}${dbSuffix}" | sed 's/^/ /' + fi + ;; + sqlite3) sqlite3 "$DATADIR/ca/ca$i/$DBNAME" 'SELECT * FROM USERS ;;' | sed 's/^/ /' + if $($FABRIC_CA_DEBUG); then + sqlite3 "$DATASRC" 'SELECT * FROM CERTIFICATES;' | sed 's/^/ /' + sqlite3 "$DATASRC" 'SELECT * FROM AFFILIATIONS;' | sed 's/^/ /' + fi + esac + done } function initFabricCa() { test -f $FABRIC_CA_SERVEREXEC || ErrorExit "fabric-ca executable not found in src tree" - $FABRIC_CA_SERVEREXEC init -c $RUNCONFIG + $FABRIC_CA_SERVEREXEC init -c $RUNCONFIG $PARENTURL $args echo "FABRIC_CA server initialized" if $($FABRIC_CA_DEBUG); then - openssl x509 -in $DST_CERT -noout -issuer -subject -serial \ + openssl x509 -in $DATADIR/$DST_CERT -noout -issuer -subject -serial \ -dates -nameopt RFC2253| sed 's/^/ /' - openssl x509 -in $DST_CERT -noout -text | + openssl x509 -in $DATADIR/$DST_CERT -noout -text | awk ' /Subject Alternative Name:/ { gsub(/^ */,"") @@ -115,9 +128,9 @@ function initFabricCa() { getline; gsub(/^ */,"") print }'| sed 's/^/ /' - openssl x509 -in $DST_CERT -noout -pubkey | + openssl x509 -in $DATADIR/$DST_CERT -noout -pubkey | openssl $KEYTYPE -pubin -noout -text 2>/dev/null| sed 's/Private/Public/' - openssl $KEYTYPE -in $DST_KEY -text 2>/dev/null + openssl $KEYTYPE -in $DATADIR/$DST_KEY -text 2>/dev/null fi } @@ -128,8 +141,6 @@ function startHaproxy() { local proxypids=$(lsof -n -i tcp | awk '$1=="haproxy" && !($2 in a) {a[$2]=$2;print a[$2]}') test -n "$proxypids" && kill $proxypids local server_port=${USER_CA_PORT-$CA_DEFAULT_PORT} - #sudo sed -i 's/ *# *$UDPServerRun \+514/$UDPServerRun 514/' /etc/rsyslog.conf - #sudo sed -i 's/ *# *$ModLoad \+imudp/$ModLoad imudp/' /etc/rsyslog.conf case $TLS_ON in "true") haproxy -f <(echo "global @@ -139,9 +150,9 @@ defaults log global option dontlognull maxconn 4096 - timeout connect 5000 - timeout client 50000 - timeout server 50000 + timeout connect 30000 + timeout client 300000 + timeout server 300000 frontend haproxy bind *:$PROXY_PORT @@ -161,6 +172,7 @@ backend fabric-cas echo " server server$i localhost:$((server_port+$i))" i=$((i+1)) done + i=0 if test -n "$FABRIC_CA_SERVER_PROFILE_PORT" ; then echo " @@ -178,8 +190,28 @@ backend fabric-ca-profile echo " server server$i localhost:$((FABRIC_CA_SERVER_PROFILE_PORT+$i))" i=$((i+1)) done + i=0 +fi + +if test -n "$FABRIC_CA_INTERMEDIATE_SERVER_PORT" ; then +echo " +frontend haproxy-intcas + bind *:$INTERMEDIATE_PROXY_PORT + mode tcp + option tcplog + default_backend fabric-intcas + +backend fabric-intcas + mode tcp + balance roundrobin"; + + while test $((i)) -lt $inst; do + echo " server intserver$i localhost:$((INTERMEDIATE_CA_DEFAULT_PORT+$i))" + i=$((i+1)) + done + i=0 fi - ) +) ;; *) haproxy -f <(echo "global @@ -191,9 +223,9 @@ defaults option httplog option dontlognull maxconn 4096 - timeout connect 5000 - timeout client 50000 - timeout server 50000 + timeout connect 30000 + timeout client 300000 + timeout server 300000 option forwardfor listen stats @@ -216,6 +248,8 @@ backend fabric-cas echo " server server$i localhost:$((server_port+$i))" i=$((i+1)) done + i=0 + if test -n "$FABRIC_CA_SERVER_PROFILE_PORT" ; then echo " frontend haproxy-profile @@ -232,11 +266,31 @@ backend fabric-ca-profile echo " server server$i localhost:$((FABRIC_CA_SERVER_PROFILE_PORT+$i))" i=$((i+1)) done + i=0 +fi + +if test -n "$FABRIC_CA_INTERMEDIATE_SERVER_PORT" ; then +echo " +frontend haproxy-intcas + bind *:$INTERMEDIATE_PROXY_PORT + mode http + option tcplog + default_backend fabric-intcas + +backend fabric-intcas + mode http + http-request set-header X-Forwarded-Port %[dst_port] + balance roundrobin"; + + while test $((i)) -lt $inst; do + echo " server intserver$i localhost:$((INTERMEDIATE_CA_DEFAULT_PORT+$i))" + i=$((i+1)) + done + i=0 fi - ) +) ;; esac - } function startFabricCa() { @@ -245,27 +299,32 @@ function startFabricCa() { local timeout="$TIMEOUT" local now=0 local server_addr=0.0.0.0 + local polladdr=$server_addr local port=${USER_CA_PORT-$CA_DEFAULT_PORT} port=$((port+$inst)) # if not explcitly set, use default - local server_port="--port $port" + test -n "${port}" && local server_port="--port $port" || local server_port="" + test -n "${CACOUNT}" && local cacount="--cacount ${CACOUNT}" if test -n "$FABRIC_CA_SERVER_PROFILE_PORT" ; then local profile_port=$((FABRIC_CA_SERVER_PROFILE_PORT+$inst)) FABRIC_CA_SERVER_PROFILE_PORT=$profile_port $FABRIC_CA_SERVEREXEC start --address $server_addr $server_port --ca.certfile $DST_CERT \ - --ca.keyfile $DST_KEY --config $RUNCONFIG 2>&1 & + --ca.keyfile $DST_KEY --config $RUNCONFIG $PARENTURL 2>&1 & else +# $FABRIC_CA_SERVEREXEC start --address $server_addr $server_port --ca.certfile $DST_CERT \ +# --ca.keyfile $DST_KEY $cacount --config $RUNCONFIG $args > $DATADIR/server${port}.log 2>&1 & $FABRIC_CA_SERVEREXEC start --address $server_addr $server_port --ca.certfile $DST_CERT \ - --ca.keyfile $DST_KEY --config $RUNCONFIG 2>&1 & + --ca.keyfile $DST_KEY $cacount --config $RUNCONFIG $args 2>&1 & fi printf "FABRIC_CA server on $server_addr:$port " + test "$server_addr" = "0.0.0.0" && polladdr="127.0.0.1" pollFabricCa "" "$server_addr" "$port" "" "$TIMEOUT" if test "$?" -eq 0; then - echo "STARTED" + echo " STARTED" else RC=$((RC+1)) - echo "FAILED" + echo " FAILED" fi } @@ -276,10 +335,11 @@ function killAllFabricCas() { test -n "$proxypids" && kill $proxypids } -while getopts "\?hRCISKXLDTAPad:t:l:n:c:k:x:g:m:p:r:o:" option; do +while getopts "\?hRCISKXLDTAPNad:t:l:n:c:k:x:g:m:p:r:o:u:U:" option; do case "$option" in a) LDAP_ENABLE="true" ;; o) TIMEOUT="$OPTARG" ;; + u) CACOUNT="$OPTARG" ;; d) DRIVER="$OPTARG" ;; r) USER_CA_PORT="$OPTARG" ;; p) HTTP_PORT="$OPTARG" ;; @@ -291,6 +351,7 @@ while getopts "\?hRCISKXLDTAPad:t:l:n:c:k:x:g:m:p:r:o:" option; do x) CA_CFG_PATH="$OPTARG" ;; m) MAXENROLL="$OPTARG" ;; g) SERVERCONFIG="$OPTARG" ;; + U) PARENTURL="$OPTARG" ;; D) export FABRIC_CA_DEBUG='true' ;; A) AUTH="false" ;; R) RESET="true" ;; @@ -301,12 +362,15 @@ while getopts "\?hRCISKXLDTAPad:t:l:n:c:k:x:g:m:p:r:o:" option; do L) LIST="true" ;; T) TLS_ON="true" ;; P) export FABRIC_CA_SERVER_PROFILE_PORT=$PROFILING_PORT ;; + N) export FABRIC_CA_INTERMEDIATE_SERVER_PORT=$INTERMEDIATE_CA_DEFAULT_PORT;; \?|h) usage exit 1 ;; esac done +shift $((OPTIND-1)) +args=$@ : ${LDAP_ENABLE:="false"} : ${TIMEOUT:=$DEFAULT_TIMEOUT} : ${HTTP_PORT:="3755"} @@ -325,7 +389,9 @@ done : ${KILL:="false"} : ${KEYTYPE:="ecdsa"} : ${KEYLEN:="256"} +: ${CACOUNT=""} test $KEYTYPE = "rsa" && SSLKEYCMD=$KEYTYPE || SSLKEYCMD="ec" +test -n "$PARENTURL" && PARENTURL="-u $PARENTURL" : ${CA_CFG_PATH:="/tmp/fabric-ca"} : ${DATADIR:="$CA_CFG_PATH"} @@ -347,15 +413,15 @@ else fi test -d $DATADIR || mkdir -p $DATADIR -DST_KEY="$DATADIR/fabric-ca-key.pem" -DST_CERT="$DATADIR/fabric-ca-cert.pem" -test -n "$SRC_CERT" && cp "$SRC_CERT" $DST_CERT -test -n "$SRC_KEY" && cp "$SRC_KEY" $DST_KEY +DST_KEY="fabric-ca-key.pem" +DST_CERT="fabric-ca-cert.pem" +test -n "$SRC_CERT" && cp "$SRC_CERT" $DATADIR/$DST_CERT +test -n "$SRC_KEY" && cp "$SRC_KEY" $DATADIR/$DST_KEY RUNCONFIG="$DATADIR/runFabricCaFvt.yaml" case $DRIVER in postgres) DATASRC="dbname=$DBNAME host=127.0.0.1 port=$POSTGRES_PORT user=postgres password=postgres sslmode=$sslmode" ;; - sqlite3) DATASRC="$DATADIR/$DBNAME" ;; + sqlite3) DATASRC="$DBNAME" ;; mysql) DATASRC="root:mysql@tcp(localhost:$MYSQL_PORT)/$DBNAME?parseTime=true$mysqlTls" ;; esac diff --git a/scripts/fvt/fabric-ca_utils b/scripts/fvt/fabric-ca_utils index a3bbfcc23..37ee6e95a 100644 --- a/scripts/fvt/fabric-ca_utils +++ b/scripts/fvt/fabric-ca_utils @@ -34,6 +34,8 @@ export DEFAULT_CA_TIMEOUT="150" export DEFAULT_HTTP_TIMEOUT="30" export DEFAULT_MSG_TIMEOUT="30" export DEFAULT_DB_TIMEOUT="30" +export INTERMEDIATE_PROXY_PORT="8054" +export INTERMEDIATE_CA_DEFAULT_PORT="1${INTERMEDIATE_PROXY_PORT}" DATE='date +%Y-%m-%d' TIME='date +%I:%M:%S%p' @@ -46,6 +48,17 @@ tolower() { echo "$1" | tr [:upper:] [:lower:] } +runPSQL() { + local cmd="$1" + local opts="$2" + local wrk_dir="$(pwd)" + cd /tmp + /usr/bin/psql "$opts" -U postgres -h localhost -c "$cmd" + local rc=$? + cd $wrk_dir + return $rc +} + setTLS() { PROTO="http://" TLSOPT="" @@ -115,13 +128,15 @@ pollServer() { : ${action:="start"} local rc=1 local starttime=$(date +%s) + local searcHost=$host + test host = "0.0.0.0" && searcHost='*' # continue to poll host:port until # we either get a response, or reach timeout while test "$(($(date +%s)-starttime))" -lt "$timeout" -a $rc -ne 0 do - printf "\r%s%03d" "Waiting for $app to $action on $host:$port ..." "$(($(date +%s)-starttime))" - ss -lpnt "src $host:$port" + printf "\r%s%03d" "Waiting for $app to $action on $host:$port... " "$(($(date +%s)-starttime))" + # ss -lpnt "src $searchHost:$port" isReachable "$host" "$port" rc=$? # invert the normal sense of 'success' for stop @@ -130,7 +145,6 @@ pollServer() { fi test $rc -eq 0 && break done - echo "Server rc $rc" return $rc } @@ -275,7 +289,6 @@ verifyServerTraffic() { END {print sessions" "status}') eval test "${result%% *}" -$op "$num_requests" 2>/dev/null; rc=$((rc+$?)) eval test "${result##* }" -$op "$percent" 2>/dev/null; rc=$((rc+$?)) - return $rc } @@ -411,7 +424,7 @@ register() { export FABRIC_CA_ENROLLMENT_DIR setTLS - $FABRIC_CA_CLIENTEXEC register -u "$PROTO${CA_HOST_ADDRESS}:$PROXY_PORT" $TLSOPT \ + $FABRIC_CA_CLIENTEXEC register -d -u "$PROTO${CA_HOST_ADDRESS}:$PROXY_PORT" $TLSOPT \ --id.name "$USERNAME" \ --id.type "$USERTYPE" \ --id.maxenrollments 1 \ @@ -449,7 +462,8 @@ function genRunconfig() { \"attrs\": { \"hf.Registrar.Roles\": \"client,user,peer,validator,auditor,ca\", \"hf.Registrar.DelegateRoles\": \"client,user,validator,auditor\", - \"hf.Revoker\": true + \"hf.Revoker\": true, + \"hf.GenCRL\": true } }, { @@ -461,7 +475,8 @@ function genRunconfig() { \"attrs\": { \"hf.Registrar.Roles\": \"client,user,peer,validator,auditor,ca\", \"hf.Registrar.DelegateRoles\": \"client,user,validator,auditor\", - \"hf.Revoker\": true + \"hf.Revoker\": true, + \"hf.GenCRL\": true } }, { @@ -675,7 +690,7 @@ done) - name: admin pass: adminpw type: client - affiliation: bank_a + affiliation: maxEnrollments: $maxEnroll attrs: hf.Registrar.Roles: \"client,user,peer,validator,auditor,ca\" @@ -683,15 +698,19 @@ done) hf.Revoker: true hf.IntermediateCA: true hf.Registrar.Attributes: \"*\" + hf.GenCRL: true - name: admin2 pass: adminpw2 type: client - affiliation: bank_a + affiliation: maxEnrollments: $maxEnroll attrs: hf.Registrar.Roles: \"client,user,peer,validator,auditor,ca\" hf.Registrar.DelegateRoles: \"client,user,validator,auditor\" hf.Revoker: true + hf.IntermediateCA: true + hf.Registrar.Attributes: \"*\" + hf.GenCRL: true - name: revoker pass: revokerpw type: client @@ -838,7 +857,7 @@ bccsp: security: 256 filekeystore: keystore: -cacount: +cacount: $CACOUNT cafiles: intermediate: parentserver: @@ -850,9 +869,10 @@ intermediate: label: tls: certfiles: + - $TLS_ROOTCERT client: - certfile: - keyfile: + certfile: $TLS_CLIENTCERT + keyfile: $TLS_CLIENTKEY EOF ;; esac diff --git a/scripts/fvt/intermediateca_test.sh b/scripts/fvt/intermediateca_test.sh index a5d8cd4ab..c82e4324f 100755 --- a/scripts/fvt/intermediateca_test.sh +++ b/scripts/fvt/intermediateca_test.sh @@ -239,7 +239,7 @@ function genIntCAConfig() { } ### Start Test ### -for driver in sqlite3 postgres mysql; do +for driver in postgres mysql; do $SCRIPTDIR/fabric-ca_setup.sh -R -x $TDIR/root -D -d $driver rm -rf $TDIR diff --git a/scripts/fvt/multica_test.sh b/scripts/fvt/multica_test.sh index 4e85c6659..a85999625 100755 --- a/scripts/fvt/multica_test.sh +++ b/scripts/fvt/multica_test.sh @@ -131,7 +131,7 @@ function resetDB() { #} ### Start Test ### -for driver in sqlite3 postgres mysql; do +for driver in postgres mysql; do # Expected codes # Result after enroll/reenroll -