diff --git a/Cargo.toml b/Cargo.toml index 1df20a3..8461679 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/gendx/stv-rs" readme = "README.md" categories = ["voting"] keywords = ["voting", "election", "stv", "meek"] -exclude = ["man/*", "testdata/*", ".github/*"] +exclude = ["man/*", "testdata/*", "tools/*", ".github/*"] edition = "2021" [dependencies] diff --git a/tools/benchmark-all-commits.sh b/tools/benchmark-all-commits.sh new file mode 100755 index 0000000..a57b1f2 --- /dev/null +++ b/tools/benchmark-all-commits.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -eux + +if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then + HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine +elif [ -e "/usr/bin/hyperfine" ]; then + HYPERFINE_PATH=/usr/bin/hyperfine +else + echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'." + exit 42 +fi + +BEGIN=main +END=HEAD + +echo "[*] Listing commits" +git log ${BEGIN}..${END} --reverse --format=oneline +COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1) + +echo "[*] Building at all commits" +./tools/build-all-commits.sh ${COMMITS} +sleep 15 + +echo "[*] Benchmarking each commit" +ITER=0 +for COMMIT in ${COMMITS} +do + ITER=$(expr ${ITER} + 1) + printf -v INDEX "%02d" ${ITER} + echo "[${INDEX}] Benchmarking ${COMMIT}" + + ./tools/benchmark-at.sh ${INDEX} ${COMMIT} 2>&1 | tee ${INDEX}-${COMMIT}.benchmark.log +done diff --git a/tools/benchmark-at.sh b/tools/benchmark-at.sh new file mode 100755 index 0000000..4c5d6c2 --- /dev/null +++ b/tools/benchmark-at.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -eux + +if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then + HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine +elif [ -e "/usr/bin/hyperfine" ]; then + HYPERFINE_PATH=/usr/bin/hyperfine +else + echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'." + exit 42 +fi + +INDEX=$1 +COMMIT=$2 +BINARY=./bin/stv-rs-${INDEX}-${COMMIT} + +if [ ! -e "${BINARY}" ]; then + echo "Binary not found at ${BINARY}. Please build it via the script at 'tools/benchmark-all-commits.sh'." + exit 42 +fi + +SLEEP_SECONDS=10 + +function benchmark() { + ARITHMETIC=$1 + EQUALIZE=$2 + INPUT=$3 + + sleep ${SLEEP_SECONDS} + ${HYPERFINE_PATH} --warmup 1 \ + "${BINARY} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=false > /dev/null" + for NUM_THREADS in 2 4 8 + do + sleep ${SLEEP_SECONDS} + RAYON_NUM_THREADS=${NUM_THREADS} ${HYPERFINE_PATH} --warmup 1 \ + "${BINARY} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=true > /dev/null" + done +} + +benchmark fixed9 "" "testdata/ballots/random/rand_hypergeometric.blt" +benchmark fixed9 "" "testdata/shuffle_ballots/rand_sorted_lexicographically.blt" +benchmark fixed9 "" "testdata/shuffle_ballots/rand_10k_sorted_lexicographically.blt" +benchmark fixed9 --equalize "testdata/ballots/random/rand_hypergeometric.blt" +#benchmark fixed9 "" "testdata/shuffle_ballots/rand_sorted_by_product.blt" +#benchmark fixed9 "" "testdata/shuffle_ballots/rand_sorted_by_lexico_product.blt" +benchmark bigfixed9 "" "testdata/ballots/random/rand_hypergeometric.blt" diff --git a/tools/benchmark-compare-commits.sh b/tools/benchmark-compare-commits.sh new file mode 100755 index 0000000..c99b57d --- /dev/null +++ b/tools/benchmark-compare-commits.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -eux + +if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then + HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine +elif [ -e "/usr/bin/hyperfine" ]; then + HYPERFINE_PATH=/usr/bin/hyperfine +else + echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'." + exit 42 +fi + +BEGIN=main +END=HEAD + +echo "[*] Listing commits" +git log ${BEGIN}..${END} --reverse --format=oneline +COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1) + +INDEX_COMMITS= +ITER=0 +for COMMIT in ${COMMITS} +do + ITER=$(expr ${ITER} + 1) + printf -v INDEX "%02d" ${ITER} + if [ "${INDEX_COMMITS}" != "" ]; then + INDEX_COMMITS="${INDEX_COMMITS}," + fi + INDEX_COMMITS=${INDEX_COMMITS}${INDEX}-${COMMIT} +done +echo "Commits joined as a hyperfine parameter list: ${INDEX_COMMITS}" + +echo "[*] Building at all commits" +./tools/build-all-commits.sh ${COMMITS} +sleep 15 + +SLEEP_SECONDS=10 + +function benchmark() { + ARITHMETIC=$1 + EQUALIZE=$2 + INPUT=$3 + + ${HYPERFINE_PATH} \ + --setup "sleep ${SLEEP_SECONDS}" \ + --warmup 1 \ + --parameter-list INDEX_COMMIT ${INDEX_COMMITS} \ + "./bin/stv-rs-{INDEX_COMMIT} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=false > /dev/null" + + for NUM_THREADS in 2 4 8 + do + RAYON_NUM_THREADS=${NUM_THREADS} ${HYPERFINE_PATH} \ + --setup "sleep ${SLEEP_SECONDS}" \ + --warmup 1 \ + --parameter-list INDEX_COMMIT ${INDEX_COMMITS} \ + "./bin/stv-rs-{INDEX_COMMIT} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=true > /dev/null" + done +} + +benchmark fixed9 "" "testdata/shuffle_ballots/rand_sorted_lexicographically.blt" +benchmark fixed9 "" "testdata/shuffle_ballots/rand_10k_sorted_lexicographically.blt" +benchmark fixed9 --equalize "testdata/ballots/random/rand_hypergeometric.blt" +benchmark bigfixed9 "" "testdata/ballots/random/rand_hypergeometric.blt" diff --git a/tools/build-all-commits.sh b/tools/build-all-commits.sh new file mode 100755 index 0000000..1652ceb --- /dev/null +++ b/tools/build-all-commits.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -eux + +COMMITS=$@ +echo "Building at commits: '${COMMITS}'" + +mkdir -p bin + +ITER=0 +for COMMIT in ${COMMITS} +do + ITER=$(expr ${ITER} + 1) + printf -v INDEX "%02d" ${ITER} + echo "[${INDEX}] Processing ${COMMIT}" + + if [ -e "bin/stv-rs-${INDEX}-${COMMIT}" ]; then + echo "Binary already exists at 'bin/stv-rs-${INDEX}-${COMMIT}', skipping..." + else + git checkout ${COMMIT} + git status + cargo +nightly build --release + cp target/release/stv-rs bin/stv-rs-${INDEX}-${COMMIT} + fi +done diff --git a/tools/disable-perf.sh b/tools/disable-perf.sh new file mode 100755 index 0000000..76ff325 --- /dev/null +++ b/tools/disable-perf.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +echo 3 > /proc/sys/kernel/perf_event_paranoid +echo 1 > /proc/sys/kernel/kptr_restrict +echo 1 > /proc/sys/kernel/nmi_watchdog diff --git a/tools/enable-perf.sh b/tools/enable-perf.sh new file mode 100755 index 0000000..50f5149 --- /dev/null +++ b/tools/enable-perf.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +echo -1 > /proc/sys/kernel/perf_event_paranoid +echo 0 > /proc/sys/kernel/kptr_restrict +echo 0 > /proc/sys/kernel/nmi_watchdog diff --git a/tools/perf-all-commits.sh b/tools/perf-all-commits.sh new file mode 100755 index 0000000..62daf28 --- /dev/null +++ b/tools/perf-all-commits.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eux + +BEGIN=main +END=HEAD + +echo "[*] Listing commits" +git log ${BEGIN}..${END} --reverse --format=oneline +COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1) + +echo "[*] Building at all commits" +./tools/build-all-commits.sh ${COMMITS} +sleep 15 + +echo "[*] Perfing each commit" +ITER=0 +for COMMIT in ${COMMITS} +do + ITER=$(expr ${ITER} + 1) + printf -v INDEX "%02d" ${ITER} + echo "[${INDEX}] Benchmarking ${COMMIT}" + + ./tools/perf-stat-at.sh ${INDEX} ${COMMIT} 2>&1 | tee ${INDEX}-${COMMIT}.perf.log +done + diff --git a/tools/perf-record.sh b/tools/perf-record.sh new file mode 100755 index 0000000..0a7bb75 --- /dev/null +++ b/tools/perf-record.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eux + +BALLOT_FILE=testdata/ballots/random/rand_hypergeometric.blt +ARITHMETIC=fixed9 + +RUSTFLAGS='-C force-frame-pointers=y' cargo +nightly build --release +BINARY=./target/release/stv-rs +sleep 5 + +function record() { + EVENT=$1 + INTERVAL=$2 + NUM_THREADS=4 + TRACE_FILE=perf-record.${NUM_THREADS}-threads.${EVENT} + + sleep 1 + RAYON_NUM_THREADS=${NUM_THREADS} perf record -e ${EVENT} -c ${INTERVAL} -g --output=${TRACE_FILE}.perf \ + ${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=true > /dev/null + perf script --input=${TRACE_FILE}.perf -F +pid > ${TRACE_FILE}.processed.perf +} + +record branch-misses 100 +record L1-dcache-load-misses 100 +record L1-icache-load-misses 100 +record LLC-loads 10 +record LLC-load-misses 1 diff --git a/tools/perf-stat-at.sh b/tools/perf-stat-at.sh new file mode 100755 index 0000000..339c5be --- /dev/null +++ b/tools/perf-stat-at.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -eux + +BALLOT_FILE=testdata/ballots/random/rand_hypergeometric.blt +ARITHMETIC=fixed9 +REPETITIONS=100 +#ARITHMETIC=bigfixed9 +#REPETITIONS=10 + +INDEX=$1 +COMMIT=$2 +BINARY=./bin/stv-rs-${INDEX}-${COMMIT} + +if [ ! -e "${BINARY}" ]; then + echo "Binary not found at ${BINARY}. Please build it via the script at 'tools/perf-all-commits.sh'." + exit 42 +fi + +SLEEP_SECONDS=5 + +function sample() { + ARITHMETIC=$1 + TITLE=$2 + PARAMS=$3 + + set +x + echo "****************************************" + echo "* ${TITLE} *" + echo "****************************************" + set -x + + sleep ${SLEEP_SECONDS} + perf stat -r ${REPETITIONS} ${PARAMS} \ + ${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=false > /dev/null + + for NUM_THREADS in 2 4 8 + do + sleep ${SLEEP_SECONDS} + RAYON_NUM_THREADS=${NUM_THREADS} perf stat -r ${REPETITIONS} ${PARAMS} \ + ${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=true > /dev/null + done +} + +function sample_default() { + sample $1 "STATS" "-d -d" +} + +function sample_events() { + sample $1 $2 "-e $3" +} + +sample_default fixed9 +sample_events fixed9 "INSTRUCTIONS" "task-clock,user_time,system_time,duration_time,cycles:u,instructions:u,branch-instructions:u,branch-misses:u" +sample_events fixed9 "L1-CACHE" "L1-dcache-loads,L1-dcache-load-misses,L1-icache-load-misses,LLC-loads" +sample_events fixed9 "LL-CACHE" "L1-dcache-loads,LLC-loads,LLC-load-misses" +#sample_events fixed9 "STORES" "L1-dcache-stores,LLC-stores,LLC-store-misses"