Add benchmarking scripts.

Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/gendx/stv-rs"
 readme = "README.md"
 categories = ["voting"]
 keywords = ["voting", "election", "stv", "meek"]
-exclude = ["man/*", "testdata/*", ".github/*"]
+exclude = ["man/*", "testdata/*", "tools/*", ".github/*"]
 edition = "2021"
 
 [dependencies]

tools/benchmark-all-commits.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -eux
+
+if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then
+    HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine
+elif [ -e "/usr/bin/hyperfine" ]; then
+    HYPERFINE_PATH=/usr/bin/hyperfine
+else
+    echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'."
+    exit 42
+fi
+
+BEGIN=main
+END=HEAD
+
+echo "[*] Listing commits"
+git log ${BEGIN}..${END} --reverse --format=oneline
+COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1)
+
+echo "[*] Building at all commits"
+./tools/build-all-commits.sh ${COMMITS}
+sleep 15
+
+echo "[*] Benchmarking each commit"
+ITER=0
+for COMMIT in ${COMMITS}
+do
+    ITER=$(expr ${ITER} + 1)
+    printf -v INDEX "%02d" ${ITER}
+    echo "[${INDEX}] Benchmarking ${COMMIT}"
+
+    ./tools/benchmark-at.sh ${INDEX} ${COMMIT} 2>&1 | tee ${INDEX}-${COMMIT}.benchmark.log
+done

tools/benchmark-at.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eux
+
+if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then
+    HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine
+elif [ -e "/usr/bin/hyperfine" ]; then
+    HYPERFINE_PATH=/usr/bin/hyperfine
+else
+    echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'."
+    exit 42
+fi
+
+INDEX=$1
+COMMIT=$2
+BINARY=./bin/stv-rs-${INDEX}-${COMMIT}
+
+if [ ! -e "${BINARY}" ]; then
+    echo "Binary not found at ${BINARY}. Please build it via the script at 'tools/benchmark-all-commits.sh'."
+    exit 42
+fi
+
+SLEEP_SECONDS=10
+
+function benchmark() {
+    ARITHMETIC=$1
+    EQUALIZE=$2
+    INPUT=$3
+
+    sleep ${SLEEP_SECONDS}
+    ${HYPERFINE_PATH} --warmup 1 \
+        "${BINARY} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=false > /dev/null"
+    for NUM_THREADS in 2 4 8
+    do
+        sleep ${SLEEP_SECONDS}
+        RAYON_NUM_THREADS=${NUM_THREADS} ${HYPERFINE_PATH} --warmup 1 \
+            "${BINARY} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=true > /dev/null"
+    done
+}
+
+benchmark fixed9    ""          "testdata/ballots/random/rand_hypergeometric.blt"
+benchmark fixed9    ""          "testdata/shuffle_ballots/rand_sorted_lexicographically.blt"
+benchmark fixed9    ""          "testdata/shuffle_ballots/rand_10k_sorted_lexicographically.blt"
+benchmark fixed9    --equalize  "testdata/ballots/random/rand_hypergeometric.blt"
+#benchmark fixed9    ""          "testdata/shuffle_ballots/rand_sorted_by_product.blt"
+#benchmark fixed9    ""          "testdata/shuffle_ballots/rand_sorted_by_lexico_product.blt"
+benchmark bigfixed9 ""          "testdata/ballots/random/rand_hypergeometric.blt"

tools/benchmark-compare-commits.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -eux
+
+if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then
+    HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine
+elif [ -e "/usr/bin/hyperfine" ]; then
+    HYPERFINE_PATH=/usr/bin/hyperfine
+else
+    echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'."
+    exit 42
+fi
+
+BEGIN=main
+END=HEAD
+
+echo "[*] Listing commits"
+git log ${BEGIN}..${END} --reverse --format=oneline
+COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1)
+
+INDEX_COMMITS=
+ITER=0
+for COMMIT in ${COMMITS}
+do
+    ITER=$(expr ${ITER} + 1)
+    printf -v INDEX "%02d" ${ITER}
+    if [ "${INDEX_COMMITS}" != "" ]; then
+        INDEX_COMMITS="${INDEX_COMMITS},"
+    fi
+    INDEX_COMMITS=${INDEX_COMMITS}${INDEX}-${COMMIT}
+done
+echo "Commits joined as a hyperfine parameter list: ${INDEX_COMMITS}"
+
+echo "[*] Building at all commits"
+./tools/build-all-commits.sh ${COMMITS}
+sleep 15
+
+SLEEP_SECONDS=10
+
+function benchmark() {
+    ARITHMETIC=$1
+    EQUALIZE=$2
+    INPUT=$3
+
+    ${HYPERFINE_PATH} \
+        --setup "sleep ${SLEEP_SECONDS}" \
+        --warmup 1 \
+        --parameter-list INDEX_COMMIT ${INDEX_COMMITS} \
+        "./bin/stv-rs-{INDEX_COMMIT} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=false > /dev/null"
+
+    for NUM_THREADS in 2 4 8
+    do
+        RAYON_NUM_THREADS=${NUM_THREADS} ${HYPERFINE_PATH} \
+            --setup "sleep ${SLEEP_SECONDS}" \
+            --warmup 1 \
+            --parameter-list INDEX_COMMIT ${INDEX_COMMITS} \
+            "./bin/stv-rs-{INDEX_COMMIT} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=true > /dev/null"
+    done
+}
+
+benchmark fixed9    ""          "testdata/shuffle_ballots/rand_sorted_lexicographically.blt"
+benchmark fixed9    ""          "testdata/shuffle_ballots/rand_10k_sorted_lexicographically.blt"
+benchmark fixed9    --equalize  "testdata/ballots/random/rand_hypergeometric.blt"
+benchmark bigfixed9 ""          "testdata/ballots/random/rand_hypergeometric.blt"

tools/build-all-commits.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -eux
+
+COMMITS=$@
+echo "Building at commits: '${COMMITS}'"
+
+mkdir -p bin
+
+ITER=0
+for COMMIT in ${COMMITS}
+do
+    ITER=$(expr ${ITER} + 1)
+    printf -v INDEX "%02d" ${ITER}
+    echo "[${INDEX}] Processing ${COMMIT}"
+
+    if [ -e "bin/stv-rs-${INDEX}-${COMMIT}" ]; then
+        echo "Binary already exists at 'bin/stv-rs-${INDEX}-${COMMIT}', skipping..."
+    else
+        git checkout ${COMMIT}
+        git status
+        cargo +nightly build --release
+        cp target/release/stv-rs bin/stv-rs-${INDEX}-${COMMIT}
+    fi
+done

tools/disable-perf.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo 3 > /proc/sys/kernel/perf_event_paranoid
+echo 1 > /proc/sys/kernel/kptr_restrict
+echo 1 > /proc/sys/kernel/nmi_watchdog

tools/enable-perf.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo -1 > /proc/sys/kernel/perf_event_paranoid
+echo 0 > /proc/sys/kernel/kptr_restrict
+echo 0 > /proc/sys/kernel/nmi_watchdog

tools/perf-all-commits.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -eux
+
+BEGIN=main
+END=HEAD
+
+echo "[*] Listing commits"
+git log ${BEGIN}..${END} --reverse --format=oneline
+COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1)
+
+echo "[*] Building at all commits"
+./tools/build-all-commits.sh ${COMMITS}
+sleep 15
+
+echo "[*] Perfing each commit"
+ITER=0
+for COMMIT in ${COMMITS}
+do
+    ITER=$(expr ${ITER} + 1)
+    printf -v INDEX "%02d" ${ITER}
+    echo "[${INDEX}] Benchmarking ${COMMIT}"
+
+    ./tools/perf-stat-at.sh ${INDEX} ${COMMIT} 2>&1 | tee ${INDEX}-${COMMIT}.perf.log
+done
+

tools/perf-record.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -eux
+
+BALLOT_FILE=testdata/ballots/random/rand_hypergeometric.blt
+ARITHMETIC=fixed9
+
+RUSTFLAGS='-C force-frame-pointers=y' cargo +nightly build --release
+BINARY=./target/release/stv-rs
+sleep 5
+
+function record() {
+    EVENT=$1
+    INTERVAL=$2
+    NUM_THREADS=4
+    TRACE_FILE=perf-record.${NUM_THREADS}-threads.${EVENT}
+
+    sleep 1
+    RAYON_NUM_THREADS=${NUM_THREADS} perf record -e ${EVENT} -c ${INTERVAL} -g --output=${TRACE_FILE}.perf \
+        ${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=true > /dev/null
+    perf script --input=${TRACE_FILE}.perf -F +pid > ${TRACE_FILE}.processed.perf
+}
+
+record branch-misses 100
+record L1-dcache-load-misses 100
+record L1-icache-load-misses 100
+record LLC-loads 10
+record LLC-load-misses 1

tools/perf-stat-at.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -eux
+
+BALLOT_FILE=testdata/ballots/random/rand_hypergeometric.blt
+ARITHMETIC=fixed9
+REPETITIONS=100
+#ARITHMETIC=bigfixed9
+#REPETITIONS=10
+
+INDEX=$1
+COMMIT=$2
+BINARY=./bin/stv-rs-${INDEX}-${COMMIT}
+
+if [ ! -e "${BINARY}" ]; then
+    echo "Binary not found at ${BINARY}. Please build it via the script at 'tools/perf-all-commits.sh'."
+    exit 42
+fi
+
+SLEEP_SECONDS=5
+
+function sample() {
+    ARITHMETIC=$1
+    TITLE=$2
+    PARAMS=$3
+
+    set +x
+    echo "****************************************"
+    echo "* ${TITLE} *"
+    echo "****************************************"
+    set -x
+
+    sleep ${SLEEP_SECONDS}
+    perf stat -r ${REPETITIONS} ${PARAMS} \
+        ${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=false > /dev/null
+
+    for NUM_THREADS in 2 4 8
+    do
+        sleep ${SLEEP_SECONDS}
+        RAYON_NUM_THREADS=${NUM_THREADS} perf stat -r ${REPETITIONS} ${PARAMS} \
+            ${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=true > /dev/null
+    done
+}
+
+function sample_default() {
+    sample $1 "STATS" "-d -d"
+}
+
+function sample_events() {
+    sample $1 $2 "-e $3"
+}
+
+sample_default fixed9
+sample_events fixed9 "INSTRUCTIONS" "task-clock,user_time,system_time,duration_time,cycles:u,instructions:u,branch-instructions:u,branch-misses:u"
+sample_events fixed9 "L1-CACHE" "L1-dcache-loads,L1-dcache-load-misses,L1-icache-load-misses,LLC-loads"
+sample_events fixed9 "LL-CACHE" "L1-dcache-loads,LLC-loads,LLC-load-misses"
+#sample_events fixed9 "STORES" "L1-dcache-stores,LLC-stores,LLC-store-misses"