chore: benchmarks [skip ci] (#155)

- lighttpd 1.4.67 (default config)
- nginx 1.22.0 (default config + worker_processes=4)
- sws 2.13.1 (default config)
- apache 2.4.54 (default config)
- caddy 2.6.1 (default config)
- binserve 0.2.0 (default config + fast_mem_cache=false, enable_hot_reload=false, enable_logging=false, enable_directory_listing=false)

Author: joseluisq

.gitignore

@@ -18,6 +18,8 @@
 **/*.out*
 **/*.perf*
 **/*empty*
+**/*.json
+**/*.csv
 release
 .vscode
 TODO
@@ -28,3 +30,4 @@ docs/*/**.html
 !sample.env
 !/docs
 !/tests/fixtures/**/*
+!benchmark/sws_benchmarks*.csv

Makefile

@@ -282,12 +282,17 @@ promote:
 .PHONY: promote
 
 loadtest:
+	@vegeta --version
 	@echo "GET http://localhost:8787" | \
-		vegeta -cpus=12 attack -workers=10 -duration=60s -connections=10000 -rate=200 -http2=false > results.bin
+		vegeta -cpus=12 attack -workers=12 -connections=500 -rate=6500/s -duration=10s -http2=false > results.bin
 	@cat results.bin | vegeta report -type='hist[0,2ms,4ms,6ms]'
 	@cat results.bin | vegeta plot > plot.html
 .PHONY: loadtest
 
+wrk:
+	@wrk -c 500 -t 12 -d 10s --latency -s benchmark/wrk_collector.lua $(WRK_URL)
+.PHONY: wrk
+
 man:
 	@asciidoctor --doctype=manpage --backend=manpage docs/man/static-web-server.1.rst
 .PHONY: man

README.md

@@ -74,6 +74,12 @@ For more details about the API, usage and examples please have a look at [The Do
 - [Release Binaries](https://github.com/joseluisq/static-web-server/releases)
 - [Platforms/Architectures Supported](https://sws.joseluisq.net/platforms-architectures/)
 
+## Benchmarks
+
+<img title="SWS - Benchmarks 2022" src="benchmark/sws_benchmarks.png" width="860">
+
+See more details on [benchmark/BENCHMARKS.md](benchmark/BENCHMARKS.md)
+
 ## Notes
 
 - If you're looking for `v1` please go to [1.x](https://github.com/joseluisq/static-web-server/tree/1.x) branch.

benchmark/BENCHMARKS.md

@@ -0,0 +1,24 @@
+# SWS - Benchmarks 2022
+
+> A benchmark suite which measures the requests per second and latency in average for several web servers.
+
+<img title="SWS - Benchmarks 2022" src="data/sws_benchmarks.png" width="860">
+
+## How to use
+
+Change `WRK_URL` with the corresponding server URL to export the wrk metrics.
+
+```sh
+WRK_URL="http://localhost" make wrk
+```
+
+## System
+
+- **OS:** Arch Linux
+- **Kernel:** 5.19.13-arch1-1 (64 bits)
+- **Processor:** 4 × Intel® Core™ i7-6500U
+- **RAM:** 8 GiB
+
+## Data
+
+For data used see [data](./data/) directory for more details.

benchmark/sws_benchmarks.csv

@@ -0,0 +1,7 @@
+server,requests,duration_ms,requests_per_sec,bytes,bytes_transfer_per_sec,connect_errors,read_errors,write_errors,http_errors,timeouts,latency_min,latency_max,latency_mean_ms,latency_stdev
+lighttpd 1.4.67,484277,"10,088.91 ms",48000.91,"1,094.53 mb",10848848.82,0,0,0,0,151,29,482359,7.36 ms,6710.11
+nginx 1.22.0,439964,"10,096.09 ms",43577.64,"3,752.89 mb",37171731.17,0,0,0,0,0,163,286376,11.44 ms,7715.38
+sws 2.13.1,413213,"10,076.99 ms",41005.59,"2,958.61 mb",29360002.27,0,0,0,0,0,74,94650,12.85 ms,9118.04
+apache 2.4.54,266118,"10,098.43 ms",26352.4,630.98 mb,6248330.38,0,5411,0,0,0,65,808201,89.82 ms,132231.85
+caddy 2.6.1,191447,"10,079.89 ms",18992.97,"3,011.53 mb",29876599.7,0,0,0,0,0,99,418180,38.44 ms,46037.13
+binserve 0.2.0,95837,"10,100.44 ms",9488.4,"1,632.43 mb",16161969.78,0,0,0,0,0,100,142686,51.15 ms,16476.27

benchmark/sws_benchmarks.png

@@ -0,0 +1,77 @@
+-- Script that runs at wrk's "done" stage collecting the stats in JSON and CSV formats.
+-- https://github.com/wg/wrk/blob/master/SCRIPTING
+done = function(summary, latency, requests)
+    local json = string.format("{\n")
+    json = json .. string.format('\t"requests": %d,\n', summary.requests)
+    json = json .. string.format('\t"duration_ms": %0.2f,\n', summary.duration / 1000)
+    json = json .. string.format('\t"requests_per_sec": %0.2f,\n', (summary.requests / summary.duration) * 1e6)
+    json = json .. string.format('\t"bytes": %d,\n', summary.bytes)
+    json = json .. string.format('\t"bytes_transfer_per_sec": %0.2f,\n', (summary.bytes / summary.duration) * 1e6)
+    json = json .. string.format('\t"connect_errors": %d,\n', summary.errors.connect)
+    json = json .. string.format('\t"read_errors": %d,\n', summary.errors.read)
+    json = json .. string.format('\t"write_errors": %d,\n', summary.errors.write)
+    json = json .. string.format('\t"http_errors": %d,\n', summary.errors.status)
+    json = json .. string.format('\t"timeouts": %d,\n', summary.errors.timeout)
+    json = json .. string.format('\t"latency_min": %0.2f,\n', latency.min)
+    json = json .. string.format('\t"latency_max": %0.2f,\n', latency.max)
+    json = json .. string.format('\t"latency_mean_ms": %0.2f,\n', latency.mean / 1000)
+    json = json .. string.format('\t"latency_stdev": %0.2f,\n', latency.stdev)
+
+    json = json .. string.format('\t"latency_distribution": [\n')
+    for _, pair in pairs({50, 75, 90, 99}) do
+        json = json .. string.format("\t\t{\n")
+
+        local percent = latency:percentile(pair)
+        json = json .. string.format('\t\t\t"percentile": %g,\n\t\t\t"latency_ms": %0.2f\n', pair, percent / 1000)
+        json = json .. string.format("\t\t}%s\n", pair > 90 and "" or ",")
+    end
+
+    json = json .. string.format("\t]\n}\n")
+
+    local file, err = io.open("benchmark/wrk_results.json", "w")
+    if file then
+        file:write(json)
+        file:close()
+    else
+        print("error saving json results file:", err)
+    end
+
+    local csv = ''
+    csv = csv .. string.format('requests,')
+    csv = csv .. string.format('duration_ms,')
+    csv = csv .. string.format('requests_per_sec,')
+    csv = csv .. string.format('bytes,')
+    csv = csv .. string.format('bytes_transfer_per_sec,')
+    csv = csv .. string.format('connect_errors,')
+    csv = csv .. string.format('read_errors,')
+    csv = csv .. string.format('write_errors,')
+    csv = csv .. string.format('http_errors,')
+    csv = csv .. string.format('timeouts,')
+    csv = csv .. string.format('latency_min,')
+    csv = csv .. string.format('latency_max,')
+    csv = csv .. string.format('latency_mean_ms,')
+    csv = csv .. string.format('latency_stdev\n')
+
+    csv = csv .. string.format('%d,', summary.requests)
+    csv = csv .. string.format('%0.2f,', summary.duration / 1000)
+    csv = csv .. string.format('%0.2f,', (summary.requests / summary.duration) * 1e6)
+    csv = csv .. string.format('%d,', summary.bytes)
+    csv = csv .. string.format('%0.2f,', (summary.bytes / summary.duration) * 1e6)
+    csv = csv .. string.format('%d,', summary.errors.connect)
+    csv = csv .. string.format('%d,', summary.errors.read)
+    csv = csv .. string.format('%d,', summary.errors.write)
+    csv = csv .. string.format('%d,', summary.errors.status)
+    csv = csv .. string.format('%d,', summary.errors.timeout)
+    csv = csv .. string.format('%0.2f,', latency.min)
+    csv = csv .. string.format('%0.2f,', latency.max)
+    csv = csv .. string.format('%0.2f,', latency.mean / 1000)
+    csv = csv .. string.format('%0.2f\n', latency.stdev)
+
+    local file, err = io.open("benchmark/wrk_results.csv", "w")
+    if file then
+        file:write(csv)
+        file:close()
+    else
+        print("error saving csv results file:", err)
+    end
+end