diff --git a/analyze.js b/analyze.js new file mode 100644 index 00000000..c09d0982 --- /dev/null +++ b/analyze.js @@ -0,0 +1,128 @@ +const fs = require('fs'); +const path = require('path'); +const exec = require('child_process').execSync; + +const extractMetric = (file, metric) => { + const content = fs.readFileSync(file, 'utf-8'); + const line = content.split('\n').find(line => line.includes(metric)); + return line ? line.split(' ')[1].replace('ms', '') : null; +}; + +const average = (values) => { + const sum = values.reduce((acc, val) => acc + parseFloat(val), 0); + return sum / values.length; +}; + +const formattedServerNames = { + tailcall: 'Tailcall', + gqlgen: 'Gqlgen', + apollo: 'Apollo GraphQL', + netflixdgs: 'Netflix DGS', + caliban: 'Caliban', + async_graphql: 'async-graphql' +}; + +const servers = ['apollo', 'caliban', 'netflixdgs', 'gqlgen', 'tailcall', 'async_graphql']; +const resultFiles = process.argv.slice(2); +const avgReqSecs = {}; +const avgLatencies = {}; + +servers.forEach((server, idx) => { + const startIdx = idx * 3; + const reqSecVals = []; + const latencyVals = []; + + for (let j = 0; j < 3; j++) { + const fileIdx = startIdx + j; + reqSecVals.push(extractMetric(resultFiles[fileIdx], 'Requests/sec')); + latencyVals.push(extractMetric(resultFiles[fileIdx], 'Latency')); + } + + avgReqSecs[server] = average(reqSecVals); + avgLatencies[server] = average(latencyVals); +}); + +// Generating data files for gnuplot +const reqSecData = '/tmp/reqSec.dat'; +const latencyData = '/tmp/latency.dat'; + +fs.writeFileSync(reqSecData, 'Server Value\n' + servers.map(server => `${server} ${avgReqSecs[server]}`).join('\n')); +fs.writeFileSync(latencyData, 'Server Value\n' + servers.map(server => `${server} ${avgLatencies[server]}`).join('\n')); + +const whichBench = resultFiles[0].startsWith('bench2') ? 2 : 1; + +const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; +const latencyHistogramFile = `latency_histogram${whichBench}.png`; + +// Plotting using gnuplot +const gnuplotScript = ` +set term pngcairo size 1280,720 enhanced font "Courier,12" +set output "${reqSecHistogramFile}" +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set title "Requests/Sec" +stats "${reqSecData}" using 2 nooutput +set yrange [0:STATS_max*1.2] +set key outside right top +plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" + +set output "${latencyHistogramFile}" +set title "Latency (in ms)" +stats "${latencyData}" using 2 nooutput +set yrange [0:STATS_max*1.2] +plot "${latencyData}" using 2:xtic(1) title "Latency" +`; + +exec(`gnuplot -e "${gnuplotScript}"`); + +const assetsDir = 'assets'; +if (!fs.existsSync(assetsDir)) { + fs.mkdirSync(assetsDir); +} + +fs.renameSync(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); +fs.renameSync(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); + +const serverRPS = {}; +servers.forEach(server => { + serverRPS[server] = avgReqSecs[server]; +}); + +const sortedServers = Object.keys(serverRPS).sort((a, b) => serverRPS[b] - serverRPS[a]); + +const resultsTable = [ + ``, + '', + '| Server | Requests/sec | Latency (ms) |', + '|--------:|--------------:|--------------:|', + ...sortedServers.map(server => { + const formattedReqSecs = parseFloat(avgReqSecs[server]).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + const formattedLatencies = parseFloat(avgLatencies[server]).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + return `| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` |`; + }), + '', + `` +].join('\n'); + +const readmeFile = 'README.md'; +const readmeContent = fs.readFileSync(readmeFile, 'utf-8'); + +if (readmeContent.includes(`PERFORMANCE_RESULTS_START_${whichBench}`)) { + const newContent = readmeContent.replace( + new RegExp(`[\\s\\S]*`, 'm'), + resultsTable + ); + fs.writeFileSync(readmeFile, newContent); +} else { + fs.appendFileSync(readmeFile, `\n${resultsTable}`); +} + +const resultsFile = 'results.md'; +fs.writeFileSync(resultsFile, `## Benchmark ${whichBench} results\n\n${resultsTable}`); + +console.log(resultsTable.replace(``, '').replace(``, '')); + +resultFiles.forEach(file => fs.unlinkSync(file)); diff --git a/analyze.sh b/analyze.sh deleted file mode 100755 index 8f515691..00000000 --- a/analyze.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash - -function extractMetric() { - local file="$1" - local metric="$2" - grep "$metric" "$file" | awk '{print $2}' | sed 's/ms//' -} - -function average() { - echo "$@" | awk '{for(i=1;i<=NF;i++) s+=$i; print s/NF}' -} - -declare -A formattedServerNames -formattedServerNames=( - ["tailcall"]="Tailcall" - ["gqlgen"]="Gqlgen" - ["apollo"]="Apollo GraphQL" - ["netflixdgs"]="Netflix DGS" - ["caliban"]="Caliban" - ["async_graphql"]="async-graphql" -) - -servers=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql") -resultFiles=("$@") -declare -A avgReqSecs -declare -A avgLatencies - -# Extract metrics and calculate averages -for idx in "${!servers[@]}"; do - startIdx=$((idx * 3)) - reqSecVals=() - latencyVals=() - for j in 0 1 2; do - fileIdx=$((startIdx + j)) - reqSecVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Requests/sec")) - latencyVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Latency")) - done - avgReqSecs[${servers[$idx]}]=$(average "${reqSecVals[@]}") - avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}") -done - -# Generating data files for gnuplot -reqSecData="/tmp/reqSec.dat" -latencyData="/tmp/latency.dat" - -echo "Server Value" > "$reqSecData" -for server in "${servers[@]}"; do - echo "$server ${avgReqSecs[$server]}" >> "$reqSecData" -done - -echo "Server Value" > "$latencyData" -for server in "${servers[@]}"; do - echo "$server ${avgLatencies[$server]}" >> "$latencyData" -done - -whichBench=1 -if [[ $1 == bench2* ]]; then - whichBench=2 -fi - -reqSecHistogramFile="req_sec_histogram${whichBench}.png" -latencyHistogramFile="latency_histogram${whichBench}.png" - -# Plotting using gnuplot -gnuplot <<- EOF - set term pngcairo size 1280,720 enhanced font "Courier,12" - set output "$reqSecHistogramFile" - set style data histograms - set style histogram cluster gap 1 - set style fill solid border -1 - set xtics rotate by -45 - set boxwidth 0.9 - set title "Requests/Sec" - stats "$reqSecData" using 2 nooutput - set yrange [0:STATS_max*1.2] - set key outside right top - plot "$reqSecData" using 2:xtic(1) title "Req/Sec" - - set output "$latencyHistogramFile" - set title "Latency (in ms)" - stats "$latencyData" using 2 nooutput - set yrange [0:STATS_max*1.2] - plot "$latencyData" using 2:xtic(1) title "Latency" -EOF - - -# Move PNGs to assets -mkdir -p assets -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Declare an associative array for server RPS -declare -A serverRPS - -# Populate the serverRPS array -for server in "${servers[@]}"; do - serverRPS[$server]=${avgReqSecs[$server]} -done - -# Get the servers sorted by RPS in descending order -IFS=$'\n' sortedServers=($(for server in "${!serverRPS[@]}"; do echo "$server ${serverRPS[$server]}"; done | sort -rn -k2 | cut -d' ' -f1)) - - -echo "Sorted servers: ${sortedServers[@]}" -# Start building the resultsTable -resultsTable="\n\n| Server | Requests/sec | Latency (ms) |\n|--------:|--------------:|--------------:|" - -# Build the resultsTable with sorted servers and formatted numbers -for server in "${sortedServers[@]}"; do - formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - resultsTable+="\n| [${formattedServerNames[$server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` |" -done - -resultsTable+="\n\n" - -echo -e $resultsTable - -# Check if the markers are present -if grep -q "PERFORMANCE_RESULTS_START_${whichBench}" README.md; then - # Replace the old results with the new results - sed -i "/PERFORMANCE_RESULTS_START_${whichBench}/,/PERFORMANCE_RESULTS_END_${whichBench}/c\\$resultsTable" README.md -else - # Append the results at the end of the README.md file - echo -e "\n$resultsTable" >> README.md -fi - -# Print the results table in a new file -resultsFile="results.md" -echo -e "## Benchmark $whichBench results\n" >> $resultsFile -echo -e $resultsTable >> $resultsFile - -# Print the results as a table in the terminal -echo -e $resultsTable | sed "s///;s///" - -# Move the generated images to the assets folder -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Delete the result TXT files -for file in "${resultFiles[@]}"; do - rm "$file" -done diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 3151eec1..e7b4e1e0 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -69,5 +69,6 @@ for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "asyn fi done -bash analyze.sh "${bench1Results[@]}" -bash analyze.sh "${bench2Results[@]}" +node analyze.js "${bench1Results[@]}" +node analyze.js "${bench2Results[@]}" +