From e7cd84e077d750abbeb7eb42e6fd50f3aa3d79f8 Mon Sep 17 00:00:00 2001 From: Jake Willey Date: Thu, 2 Dec 2021 06:59:26 -0800 Subject: [PATCH 1/4] Adding loop script and updating the run log to include the new query operations --- .../Tools/Benchmark/loop.sh | 35 +++++++++++++++ .../Tools/Benchmark/run.sh | 45 ++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh new file mode 100644 index 0000000000..f87736a15f --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export OSSProjectRef=True +export RESULTS_PK=test_runs +export PL=18 + +#These must be configured +export ACCOUNT_ENDPOINT= +export ACCOUNT_KEY= +export TELEMETRY_ENDPOINT= + +#Loop forever +i=0 +while : +do + #Kill any running processes + pkill -f run.sh + git pull origin --depth=1 + + # Query operations take a long time + # Only run it them once every 10 runs + if [ $(expr $i % 10) != "0" ]; then + export INCLUDE_QUERY = true + else + export INCLUDE_QUERY = false + fi + i++ + + ./run.sh + + echo "====== Waiting for 10Sec =================" + sleep 10 #Wait for 10sec + +done + diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh index d0d9520b78..fa30b344be 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh @@ -24,8 +24,49 @@ then exit -1 fi -for BENCHMARK_NAME in InsertV3 ReadStreamExistsV3 #ReadFeedStreamV3 ReadNotExistsV3 ReadTExistsV3 +if [ -z "$TELEMETRY_ENDPOINT" ] +then + echo "Missing TELEMETRY_ENDPOINT" + exit -1 +fi + +if [ -z "$INCLUDE_QUERY" ] +then + echo "Missing INCLUDE_QUERY" + exit -1 +fi + +CommitId = $(git log -1 | head -n 1 | cut -d ' ' -f 2) +CommitDate = $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 1 -d ' ') +CommitTime = $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 2 -d ' ') +Branchname = $(git rev-parse --abbrev-ref HEAD) + +# Client telemetry disabled ReadStreamExistsV3 +dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk +sleep 10 #Wait + +# Client telemetry enabled ReadStreamExistsV3. This is needed to see the impact of client telemetry. +dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk +sleep 10 #Wait + +#Point read operations +for WORKLOAD_NAME in ReadNotExistsV3 ReadTExistsV3 ReadStreamExistsWithDiagnosticsV3 do - dotnet run -c Release -- -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $(git log -1 | head -n 1 | cut -d ' ' -f 2) --commitdate $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 1 -d ' ') --committime $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 2 -d ' ') --branchname $(git rev-parse --abbrev-ref HEAD) --database testdb --container testcol --partitionkeypath /pk -n 2000000 -w $BENCHMARK_NAME --pl $PL + dotnet run -c Release -- -n 2000000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk + sleep 10 #Wait done +#Insert operation +dotnet run -c Release -- -n 2000000 -w InsertV3 --pl 30 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 1 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk +sleep 45 #Wait + +if [ "$INCLUDE_QUERY" = true ] +then + #Query operations + # n value is lowered to 100000 because queries are significantly slower. This prevents the runs from taking to long. + for WORKLOAD_NAME in ReadFeedStreamV3 QueryTSinglePkV3 QueryTSinglePkOrderByWithPaginationV3 QueryTSinglePkOrderByFullDrainV3 QueryTCrossPkV3 QueryTCrossPkOrderByWithPaginationV3 QueryTCrossPkOrderByFullDrainV3 QueryStreamSinglePkV3 QueryStreamSinglePkOrderByWithPaginationV3 QueryStreamSinglePkOrderByFullDrainV3 QueryStreamCrossPkV3 QueryStreamCrossPkOrderByWithPaginationV3 QueryStreamCrossPkOrderByFullDrainV3 + $do + dotnet run -c Release -- -n 100000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY--enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk + sleep 10 #Wait + done +fi \ No newline at end of file From e1034fbb1094d719f4b323503e43cb3b6f79effa Mon Sep 17 00:00:00 2001 From: Jake Willey Date: Thu, 2 Dec 2021 11:24:46 -0800 Subject: [PATCH 2/4] Fix scripts --- .../Tools/Benchmark/loop.sh | 16 ++++++------- .../Tools/Benchmark/run.sh | 23 +++++++++++-------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh index f87736a15f..ae2d8eddf5 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh @@ -5,7 +5,7 @@ export RESULTS_PK=test_runs export PL=18 #These must be configured -export ACCOUNT_ENDPOINT= +export ACCOUNT_ENDPOINT= export ACCOUNT_KEY= export TELEMETRY_ENDPOINT= @@ -18,18 +18,18 @@ do git pull origin --depth=1 # Query operations take a long time - # Only run it them once every 10 runs - if [ $(expr $i % 10) != "0" ]; then - export INCLUDE_QUERY = true + # Only run them once every 10 runs + if [ $(($i % 10)) -eq 0 ]; then + echo Query run is enabled + export INCLUDE_QUERY=true else - export INCLUDE_QUERY = false + export INCLUDE_QUERY=false fi - i++ + ((i++)) ./run.sh echo "====== Waiting for 10Sec =================" sleep 10 #Wait for 10sec -done - +done \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh index fa30b344be..9a2aaf5e4b 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh @@ -36,28 +36,33 @@ then exit -1 fi -CommitId = $(git log -1 | head -n 1 | cut -d ' ' -f 2) -CommitDate = $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 1 -d ' ') -CommitTime = $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 2 -d ' ') -Branchname = $(git rev-parse --abbrev-ref HEAD) +COMMIT_ID=$(git log -1 | head -n 1 | cut -d ' ' -f 2) +COMMIT_DATE=$(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 1 -d ' ') +COMMIT_TIME=$(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 2 -d ' ') +BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) + +echo $COMMIT_ID +echo $COMMIT_DATE +echo $COMMIT_TIME +echo $BRANCH_NAME # Client telemetry disabled ReadStreamExistsV3 -dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk +dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk sleep 10 #Wait # Client telemetry enabled ReadStreamExistsV3. This is needed to see the impact of client telemetry. -dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk +dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk sleep 10 #Wait #Point read operations for WORKLOAD_NAME in ReadNotExistsV3 ReadTExistsV3 ReadStreamExistsWithDiagnosticsV3 do - dotnet run -c Release -- -n 2000000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk + dotnet run -c Release -- -n 2000000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk sleep 10 #Wait done #Insert operation -dotnet run -c Release -- -n 2000000 -w InsertV3 --pl 30 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 1 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk +dotnet run -c Release -- -n 2000000 -w InsertV3 --pl 30 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 1 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk sleep 45 #Wait if [ "$INCLUDE_QUERY" = true ] @@ -66,7 +71,7 @@ then # n value is lowered to 100000 because queries are significantly slower. This prevents the runs from taking to long. for WORKLOAD_NAME in ReadFeedStreamV3 QueryTSinglePkV3 QueryTSinglePkOrderByWithPaginationV3 QueryTSinglePkOrderByFullDrainV3 QueryTCrossPkV3 QueryTCrossPkOrderByWithPaginationV3 QueryTCrossPkOrderByFullDrainV3 QueryStreamSinglePkV3 QueryStreamSinglePkOrderByWithPaginationV3 QueryStreamSinglePkOrderByFullDrainV3 QueryStreamCrossPkV3 QueryStreamCrossPkOrderByWithPaginationV3 QueryStreamCrossPkOrderByFullDrainV3 $do - dotnet run -c Release -- -n 100000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY--enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $CommitId --commitdate $Commitdate --committime $Committime --branchname $Branchname --database testdb --container testcol --partitionkeypath /pk + dotnet run -c Release -- -n 200000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY--enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk sleep 10 #Wait done fi \ No newline at end of file From 40a74364c6d91b66aa9b3cf04319b40b672297ff Mon Sep 17 00:00:00 2001 From: Jake Willey Date: Fri, 3 Dec 2021 12:07:57 -0800 Subject: [PATCH 3/4] Add different name --- Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh index 9a2aaf5e4b..bfdb1b9221 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh @@ -51,7 +51,7 @@ dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --tcp 10 --pl $PL -e sleep 10 #Wait # Client telemetry enabled ReadStreamExistsV3. This is needed to see the impact of client telemetry. -dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk +dotnet run -c Release -- -n 2000000 -w ReadStreamExistsV3 --WorkloadName ReadStreamExistsV3WithTelemetry --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk sleep 10 #Wait #Point read operations From 34fd920e5bdb6e93401b64a90cdcc0b699f823c5 Mon Sep 17 00:00:00 2001 From: Jake Willey Date: Mon, 6 Dec 2021 11:59:17 -0800 Subject: [PATCH 4/4] Update scripts with more fixes --- Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh | 2 +- Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh index ae2d8eddf5..845f1c0022 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh @@ -15,7 +15,7 @@ while : do #Kill any running processes pkill -f run.sh - git pull origin --depth=1 + git pull origin master # Query operations take a long time # Only run them once every 10 runs diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh index bfdb1b9221..96b3a73832 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh @@ -71,7 +71,7 @@ then # n value is lowered to 100000 because queries are significantly slower. This prevents the runs from taking to long. for WORKLOAD_NAME in ReadFeedStreamV3 QueryTSinglePkV3 QueryTSinglePkOrderByWithPaginationV3 QueryTSinglePkOrderByFullDrainV3 QueryTCrossPkV3 QueryTCrossPkOrderByWithPaginationV3 QueryTCrossPkOrderByFullDrainV3 QueryStreamSinglePkV3 QueryStreamSinglePkOrderByWithPaginationV3 QueryStreamSinglePkOrderByFullDrainV3 QueryStreamCrossPkV3 QueryStreamCrossPkOrderByWithPaginationV3 QueryStreamCrossPkOrderByFullDrainV3 $do - dotnet run -c Release -- -n 200000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY--enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk + dotnet run -c Release -- -n 200000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk sleep 10 #Wait done fi \ No newline at end of file