Skip to content

Commit

Permalink
[FAB-4764] Rearrange Ledger test parameter setup
Browse files Browse the repository at this point in the history 
Increase LTE tool usability by rearranging the Ledger performance test
parameter setup so that the parameter file has more control on the test
run. Also, make the test driver take the parameter file as a (required)
option instead of being hardcoded.

Modifies the README.md and the python test wrapper to reflect these
changes.

Change-Id: I2cbdb0d082b3b8f9f143d2f346742c84556e3c3a
Signed-off-by: Adnan Choudhury <adnan.choudhury@itpeoplecorp.com>
  • Loading branch information
@adnan-c
adnan-c committed Jun 21, 2017
1 parent a5b1ed3 commit 74d1652
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 43 deletions.
34 changes: 18 additions & 16 deletions test/regression/daily/ledger_lte.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import unittest
import subprocess

tool_directory = '../../tools/LTE/scripts'

class perf_goleveldb(unittest.TestCase):

def test_FAB_3790_VaryNumParallelTxPerChain(self):
Expand All @@ -20,9 +22,9 @@ def test_FAB_3790_VaryNumParallelTxPerChain(self):
'''
logfile = open("output_VaryNumParallelTxPerChain.log", "w")
returncode = subprocess.call(
"./runbenchmarks.sh varyNumParallelTxPerChain",
"./runbenchmarks.sh -f parameters_daily_CI.sh varyNumParallelTxPerChain",
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
cwd='../../tools/LTE/scripts')
cwd=tool_directory)
logfile.close()
self.assertEqual(returncode, 0, msg="VaryNumParallelTxPerChain "
"performance test failed. \nPlease check the logfile "
Expand All @@ -40,9 +42,9 @@ def test_FAB_3795_VaryNumChains(self):
'''
logfile = open("output_VaryNumChains.log", "w")
returncode = subprocess.call(
"./runbenchmarks.sh varyNumChains",
"./runbenchmarks.sh -f parameters_daily_CI.sh varyNumChains",
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
cwd='../../tools/LTE/scripts')
cwd=tool_directory)
logfile.close()
self.assertEqual(returncode, 0, msg="VaryNumChains performance test"
" failed. \nPlease check the logfile "+logfile.name+" for more "
Expand All @@ -60,9 +62,9 @@ def test_FAB_3798_VaryNumParallelTxWithSingleChain(self):
'''
logfile = open("output_VaryNumParallelTxWithSingleChain.log", "w")
returncode = subprocess.call(
"./runbenchmarks.sh varyNumParallelTxWithSingleChain",
"./runbenchmarks.sh -f parameters_daily_CI.sh varyNumParallelTxWithSingleChain",
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
cwd='../../tools/LTE/scripts')
cwd=tool_directory)
logfile.close()
self.assertEqual(returncode, 0, msg="VaryNumParallelTxWithSingleChain "
"performance test failed. \nPlease check the logfile "
Expand All @@ -80,9 +82,9 @@ def test_FAB_3799_VaryNumChainsWithNoParallelism(self):
'''
logfile = open("output_VaryNumChainsWithNoParallelism.log", "w")
returncode = subprocess.call(
"./runbenchmarks.sh varyNumChainsWithNoParallelism",
"./runbenchmarks.sh -f parameters_daily_CI.sh varyNumChainsWithNoParallelism",
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
cwd='../../tools/LTE/scripts')
cwd=tool_directory)
logfile.close()
self.assertEqual(returncode, 0, msg="varyNumChainsWithNoParallelism "
"performance test failed. \nPlease check the logfile "
Expand All @@ -99,9 +101,9 @@ def test_FAB_3801_VaryKVSize(self):
'''
logfile = open("output_VaryKVSize.log", "w")
returncode = subprocess.call(
"./runbenchmarks.sh varyKVSize",
"./runbenchmarks.sh -f parameters_daily_CI.sh varyKVSize",
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
cwd='../../tools/LTE/scripts')
cwd=tool_directory)
logfile.close()
self.assertEqual(returncode, 0, msg="varyKVSize performance test"
" failed. \nPlease check the logfile "+logfile.name+" for more "
Expand All @@ -119,9 +121,9 @@ def test_FAB_3802_VaryBatchSize(self):
'''
logfile = open("output_VaryBatchSize.log", "w")
returncode = subprocess.call(
"./runbenchmarks.sh varyBatchSize",
"./runbenchmarks.sh -f parameters_daily_CI.sh varyBatchSize",
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
cwd='../../tools/LTE/scripts')
cwd=tool_directory)
logfile.close()
self.assertEqual(returncode, 0, msg="varyBatchSize performance test"
" failed. \nPlease check the logfile "+logfile.name+" for more "
Expand All @@ -139,9 +141,9 @@ def test_FAB_3800_VaryNumKeysInEachTx(self):
'''
logfile = open("output_VaryNumKeysInEachTx.log", "w")
returncode = subprocess.call(
"./runbenchmarks.sh varyNumKeysInEachTx",
"./runbenchmarks.sh -f parameters_daily_CI.sh varyNumKeysInEachTx",
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
cwd='../../tools/LTE/scripts')
cwd=tool_directory)
logfile.close()
self.assertEqual(returncode, 0, msg="varyNumKeysInEachTx performance "
"test failed. \nPlease check the logfile "+logfile.name
Expand All @@ -159,9 +161,9 @@ def test_FAB_3803_VaryNumTxs(self):
'''
logfile = open("output_VaryNumTxs.log", "w")
returncode = subprocess.call(
"./runbenchmarks.sh varyNumTxs",
"./runbenchmarks.sh -f parameters_daily_CI.sh varyNumTxs",
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
cwd='../../tools/LTE/scripts')
cwd=tool_directory)
logfile.close()
self.assertEqual(returncode, 0, msg="varyNumTxs performance test"
" failed. \nPlease check the logfile "+logfile.name+" for more "
Expand Down
20 changes: 15 additions & 5 deletions test/tools/LTE/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,15 @@ considered to constitute a single test-run.

## How to Run The tests

To run all the available tests with default parameters, run:
To run all the available tests, run:
```
cd fabric/test/tools/LTE/scripts
./runbenchmark.sh all
./runbenchmark.sh -f parameters_daily_CI.sh all
```
you can run individual tests without running all the available tests by giving
where the file `parameters_daily_CI.sh` has all the necessary test parameters.


You can run individual tests without running all the available tests by giving
the name of the test as parameter, instead of `all`. You can get the available
test names by:
```
Expand All @@ -35,7 +38,7 @@ test names by:

### What the Tests Do

Each test reads test parameters from the file `parameters_daily_CI.sh` and
Each test reads test parameters from the provided parameter file and
conducts several test-runs by varying one or two of the parameters. The
parameters are:
* number of chains (ledger),
Expand Down Expand Up @@ -95,8 +98,15 @@ transactions on chain_2 in parallel In each of the transactions executed by any
client, the transaction expects and modifies any key(s) between Key_1 to key_50
(because, total keys are to be 100 across two chains).

### Running with Custom Parameters

The tests can be run with user-defined parameters by creating a new file that
has all the necessary parameters to run and using that file as the input (see
the section on how to run the tests) . The names of necessary parameters can be
found in the file `parameters_daily_CI.sh`.

## How to View the Test Results

The test results can be viewed as in the stdout where it shows how long each
single operation took to complete in a test. These results are also saved in a
.csv file in the following directory: `/tmp/fabric/test/tools/LTE/results`
.csv file in the following directory: `/tmp/experiments`
26 changes: 24 additions & 2 deletions test/tools/LTE/scripts/parameters_daily_CI.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,30 @@
DataDir="/tmp/fabric/test/tools/LTE/data"
NumChains=10
NumParallelTxPerChain=10
NumKVs=1000000
NumTotalTx=1000000
NumKVs=100000
NumTotalTx=100000
NumKeysInEachTx=4
BatchSize=50
KVSize=200

# Each test consists of several test-runs, where one single parameter is varied
# between the test-runs and rest of the parameters remain same. Each array below
# holds the values of that parameter for each corresponding test. For example,
# the "ArrayNumChains" will be used in the test titled "varyNumChains", and the
# 8 test-runs that exist for testing the varying number of chains will use these
# values:
# NumChains = (1 5 10 20 50 100 500 2000)
# NumParallelTxPerChain=10
# NumKVs=100000
# NumTotalTx=100000
# NumKeysInEachTx=4
# BatchSize=50
# KVSize=200
ArrayNumParallelTxPerChain=(1 5 10 20 50 100 500 2000)
ArrayNumChains=(1 5 10 20 50 100 500 2000)
ArrayNumKeysInEachTx=(1 2 5 10 20)
ArrayKVSize=(100 200 500 1000 2000)
ArrayBatchSize=(10 20 100 500)
ArrayNumParallelTxWithSingleChain=(1 5 10 20 50 100 500 2000)
ArrayNumChainsWithNoParallelism=(1 5 10 20 50 100 500 2000)
ArrayNumTxs=(100000 200000 500000 1000000)
74 changes: 54 additions & 20 deletions test/tools/LTE/scripts/runbenchmarks.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,66 +6,72 @@
#

source ./benchmarks.sh
source ./parameters_daily_CI.sh

########################################################################################################
# This shell script contains a series of benchmark tests
# The test parameters are imported from "parameters_daily_CI.sh"
########################################################################################################

function varyNumParallelTxPerChain {
for v in 1 5 10 20 50 100 500 2000; do
for v in "${ArrayNumParallelTxPerChain[@]}"
do
NumParallelTxPerChain=$v
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
done
}

function varyNumChains {
for v in 1 5 10 20 50 100 500 2000; do
for v in "${ArrayNumChains[@]}"
do
NumChains=$v
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
done
}

function varyNumKeysInEachTx {
for v in 1 2 5 10 20; do
for v in "${ArrayNumKeysInEachTx[@]}"
do
NumKeysInEachTx=$v
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
done
}

function varyKVSize {
for v in 100 200 500 1000 2000; do
for v in "${ArrayKVSize[@]}"
do
KVSize=$v
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
done
}

function varyBatchSize {
for v in 10 20 100 500; do
for v in "${ArrayBatchSize[@]}"
do
BatchSize=$v
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
done
}

function varyNumParallelTxWithSingleChain {
NumChains=1
for v in 1 5 10 20 50 100 500 2000; do
for v in "${ArrayNumParallelTxWithSingleChain[@]}"
do
NumParallelTxPerChain=$v
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
done
}

function varyNumChainsWithNoParallelism {
NumParallelTxPerChain=1
for v in 1 5 10 20 50 100 500 2000; do
for v in "${ArrayNumChainsWithNoParallelism[@]}"
do
NumChains=$v
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
done
}

function varyNumTxs {
for v in 1000000 2000000 5000000 10000000; do
for v in "${ArrayNumTxs[@]}"
do
NumTotalTx=$v
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
done
Expand All @@ -77,6 +83,42 @@ function runLargeDataExperiment {
rm -rf $DataDir;runInsertTxs;runReadWriteTxs
}

function usage () {
printf "Usage: ./runbenchmarks.sh [-f parameter_file_name] [test_name]\nAvailable tests (use \"all\" to run all tests):
varyNumParallelTxPerChain
varyNumChains
varyNumParallelTxWithSingleChain
varyNumChainsWithNoParallelism
varyNumKeysInEachTx
varyKVSize
varyBatchSize
varyNumTxs
runLargeDataExperiment\n"
}

PARAM_FILE=""

while getopts ":f:" opt; do
case $opt in
f)
printf "Parameter file: $OPTARG"
PARAM_FILE=$OPTARG;;
\?)
printf "Error: invalid parameter -$OPTARG!" >> /dev/stderr
usage
exit 1;;
esac
done

if [ ! $PARAM_FILE ]
then
printf "Error: No Parameter file given!" >> /dev/stderr
usage
exit 1
else
source $PARAM_FILE
fi

shift $(expr $OPTIND - 1 )

case $1 in
Expand All @@ -99,16 +141,7 @@ case $1 in
runLargeDataExperiment)
runLargeDataExperiment ;;
help)
printf "Usage: ./runbenchmarks.sh [test_name]\nAvailable tests (use \"all\" to run all tests):
varyNumParallelTxPerChain
varyNumChains
varyNumParallelTxWithSingleChain
varyNumChainsWithNoParallelism
varyNumKeysInEachTx
varyKVSize
varyBatchSize
varyNumTxs
runLargeDataExperiment\n"
usage
;;
all)
varyNumParallelTxPerChain
Expand All @@ -122,5 +155,6 @@ runLargeDataExperiment\n"
runLargeDataExperiment ;;
*)
printf "Error: test name empty/incorrect!\n" >> /dev/stderr
usage
exit 1 ;;
esac

0 comments on commit 74d1652

Please sign in to comment.