Add websockets benchmark (#422)

emmett-framework · Nov 7, 2024 · 2e235d1 · 2e235d1
1 parent 261ceba
commit 2e235d1
Show file tree

Hide file tree

Showing 11 changed files with 445 additions and 75 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -92,6 +92,38 @@ jobs:
         name: results-vs
         path: benchmarks/results/*
 
+  benchmark-ws:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - uses: pyo3/maturin-action@v1
+      with:
+        command: build
+        args: --release --interpreter python3.11
+        target: x64
+        manylinux: auto
+        container: off
+    - run: |
+        export _whl=$(ls target/wheels/granian-*.whl)
+        pip install $_whl
+    - name: deps
+      run: |
+        pip install -r benchmarks/envs/asgi.txt
+        pip install websockets
+    - name: benchmark
+      working-directory: ./benchmarks
+      run: |
+        python benchmarks.py vs_ws
+    - name: upload results
+      uses: actions/upload-artifact@v4
+      with:
+        name: results-ws
+        path: benchmarks/results/*
+
   benchmark-pyver:
     runs-on: ubuntu-latest
     needs: [toolchain]
@@ -146,7 +178,7 @@ jobs:
 
   results:
     runs-on: ubuntu-latest
-    needs: [benchmark-base, benchmark-vs, benchmark-pyver]
+    needs: [benchmark-base, benchmark-vs, benchmark-ws, benchmark-pyver]
 
     steps:
     - uses: actions/checkout@v4
@@ -163,6 +195,12 @@ jobs:
         path: benchmarks/results
     - run: |
         mv benchmarks/results/data.json benchmarks/results/vs.json
+    - uses: actions/download-artifact@v4
+      with:
+        name: results-ws
+        path: benchmarks/results
+    - run: |
+        mv benchmarks/results/data.json benchmarks/results/ws.json
     - uses: actions/download-artifact@v4
       with:
         name: results-pyver
@@ -171,7 +209,7 @@ jobs:
       working-directory: ./benchmarks
       run: |
         noir -c data:results/base.json -v 'benv=GHA Linux x86_64' templates/main.md > README.md
-        noir -c data:results/vs.json -v 'benv=GHA Linux x86_64' templates/vs.md > vs.md
+        noir -c data:results/vs.json -c wsdata:results/ws.json -v 'benv=GHA Linux x86_64' templates/vs.md > vs.md
         noir \
           -c data310:results/py310.json \
           -c data311:results/py311.json \

diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -2,10 +2,10 @@
 
 
 
-Run at: Tue 03 Sep 2024, 21:44    
+Run at: Mon 28 Oct 2024, 02:09    
 Environment: GHA Linux x86_64 (CPUs: 4)    
 Python version: 3.11    
-Granian version: 1.6.0    
+Granian version: 1.6.2    
 
 ## RSGI response types
 
@@ -14,10 +14,10 @@ The "small" response is 4 bytes, the "big" one is 80kbytes.
 
 | Type | Total requests | RPS | avg latency | max latency |
 | --- | --- | --- | --- | --- |
-| bytes small (c64) | 413639 | 41365 | 1.544ms | 4.612ms |
-| str small (c128) | 433016 | 43346 | 2.944ms | 19.673ms |
-| bytes big (c64) | 281576 | 28162 | 2.267ms | 5.781ms |
-| str big (c64) | 288430 | 28851 | 2.211ms | 6.574ms |
+| bytes small (c256) | 392764 | 39392 | 6.47ms | 61.121ms |
+| str small (c512) | 424544 | 42723 | 11.921ms | 130.529ms |
+| bytes big (c64) | 288745 | 28874 | 2.212ms | 6.843ms |
+| str big (c64) | 276754 | 27681 | 2.305ms | 5.861ms |
 
 
 ## Interfaces
@@ -29,15 +29,15 @@ The "echo" request is a 4bytes POST request responding with the same body.
 
 | Request | Total requests | RPS | avg latency | max latency |
 | --- | --- | --- | --- | --- |
-| RSGI bytes (c128) | 423456 | 42379 | 3.009ms | 27.917ms |
-| RSGI str (c128) | 428954 | 42942 | 2.972ms | 24.262ms |
-| RSGI echo (c256) | 377133 | 37812 | 6.735ms | 74.44ms |
-| ASGI bytes (c64) | 436301 | 43632 | 1.463ms | 4.525ms |
-| ASGI str (c64) | 422612 | 42269 | 1.51ms | 4.182ms |
-| ASGI echo (c512) | 217386 | 21864 | 23.309ms | 121.784ms |
-| WSGI bytes (c128) | 367719 | 36827 | 3.463ms | 18.241ms |
-| WSGI str (c64) | 373343 | 37340 | 1.71ms | 4.425ms |
-| WSGI echo (c64) | 342874 | 34293 | 1.861ms | 4.515ms |
+| RSGI bytes (c64) | 397815 | 39792 | 1.602ms | 4.506ms |
+| RSGI str (c64) | 414448 | 41451 | 1.541ms | 4.152ms |
+| RSGI echo (c256) | 366975 | 36780 | 6.93ms | 59.373ms |
+| ASGI bytes (c512) | 394155 | 39596 | 12.855ms | 143.895ms |
+| ASGI str (c64) | 402564 | 40258 | 1.585ms | 3.888ms |
+| ASGI echo (c128) | 217107 | 21742 | 5.864ms | 31.625ms |
+| WSGI bytes (c128) | 375703 | 37613 | 3.386ms | 39.83ms |
+| WSGI str (c256) | 371867 | 37268 | 6.837ms | 58.755ms |
+| WSGI echo (c128) | 352872 | 35325 | 3.609ms | 33.688ms |
 
 
 ## HTTP/2
@@ -46,10 +46,10 @@ Comparison between Granian HTTP versions on RSGI using 4bytes plain text respons
 
 | Request | Total requests | RPS | avg latency | max latency |
 | --- | --- | --- | --- | --- |
-| HTTP/1 [GET] (c128) | 409169 | 40964 | 3.11ms | 35.937ms |
-| HTTP/1 [POST] (c256) | 367186 | 36816 | 6.927ms | 63.568ms |
-| HTTP/2 [GET] (c64) | 429019 | 42908 | 1.489ms | 6.358ms |
-| HTTP/2 [POST] (c64) | 279182 | 27921 | 2.287ms | 6.201ms |
+| HTTP/1 [GET] (c64) | 402839 | 40293 | 1.584ms | 3.983ms |
+| HTTP/1 [POST] (c64) | 361471 | 36152 | 1.766ms | 5.997ms |
+| HTTP/2 [GET] (c128) | 411331 | 41181 | 3.097ms | 35.375ms |
+| HTTP/2 [POST] (c256) | 285962 | 28684 | 8.881ms | 102.249ms |
 
 
 ## File responses
@@ -59,15 +59,16 @@ WSGI is not part of the benchmark since the protocol doesn't implement anything
 
 | Request | Total requests | RPS | avg latency | max latency |
 | --- | --- | --- | --- | --- |
-| RSGI (c64) | 340095 | 34014 | 1.877ms | 3.9ms |
-| ASGI (c256) | 168845 | 16957 | 14.998ms | 81.803ms |
-| ASGI pathsend (c64) | 299960 | 29998 | 2.129ms | 4.663ms |
+| RSGI (c64) | 336910 | 33698 | 1.894ms | 9.23ms |
+| ASGI (c64) | 167204 | 16723 | 3.818ms | 5.76ms |
+| ASGI pathsend (c128) | 290053 | 29038 | 4.392ms | 28.584ms |
 
 
 ### Other benchmarks
 
 - [Versus 3rd party servers](./vs.md)
 - [Concurrency](./concurrency.md)
+- [Python versions](./pyver.md)
 
 ### 3rd party benchmarks
 

diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py
@@ -11,37 +11,38 @@
 
 CPU = multiprocessing.cpu_count()
 WRK_CONCURRENCIES = [64, 128, 256, 512]
+WS_CONCURRENCIES = [(8, 20_000), (16, 10_000), (32, 5000), (64, 2500)]
 
 APPS = {
     'asgi': (
         'granian --interface asgi --log-level warning --backlog 2048 '
-        '--no-ws --http {http} '
+        '{wsmode}--http {http} '
         '--workers {procs} --threads {threads}{bthreads} '
-        '--threading-mode {thmode} app.asgi:app'
+        '--threading-mode {thmode} {app}.asgi:app'
     ),
     'rsgi': (
         'granian --interface rsgi --log-level warning --backlog 2048 '
-        '--no-ws --http {http} '
+        '{wsmode}--http {http} '
         '--workers {procs} --threads {threads}{bthreads} '
-        '--threading-mode {thmode} app.rsgi:app'
+        '--threading-mode {thmode} {app}.rsgi:app'
     ),
     'wsgi': (
         'granian --interface wsgi --log-level warning --backlog 2048 '
-        '--no-ws --http {http} '
+        '{wsmode}--http {http} '
         '--workers {procs} --threads {threads}{bthreads} '
         '--threading-mode {thmode} app.wsgi:app'
     ),
     'uvicorn_h11': (
-        'uvicorn --interface asgi3 --no-access-log --log-level warning --http h11 --workers {procs} app.asgi:app'
+        'uvicorn --interface asgi3 --no-access-log --log-level warning --http h11 --workers {procs} {app}.asgi:app'
     ),
     'uvicorn_httptools': (
         'uvicorn --interface asgi3 '
         '--no-access-log --log-level warning '
-        '--http httptools --workers {procs} app.asgi:app'
+        '--http httptools --workers {procs} {app}.asgi:app'
     ),
     'hypercorn': (
         'hypercorn -b localhost:8000 -k uvloop --log-level warning --backlog 2048 '
-        '--workers {procs} asgi:app.asgi:async_app'
+        '--workers {procs} asgi:{app}.asgi:async_app'
     ),
     'gunicorn_gthread': 'gunicorn --workers {procs} -k gthread app.wsgi:app',
     'gunicorn_gevent': 'gunicorn --workers {procs} -k gevent app.wsgi:app',
@@ -54,18 +55,21 @@
 
 
 @contextmanager
-def app(name, procs=None, threads=None, bthreads=None, thmode=None, http='1'):
+def app(name, procs=None, threads=None, bthreads=None, thmode=None, http='1', ws=False, app_path='app'):
     procs = procs or 1
     threads = threads or 1
     bthreads = f' --blocking-threads {bthreads}' if bthreads else ''
     thmode = thmode or 'workers'
+    wsmode = '--no-ws ' if not ws else ''
     exc_prefix = os.environ.get('BENCHMARK_EXC_PREFIX')
     proc_cmd = APPS[name].format(
+        app=app_path,
         procs=procs,
         threads=threads,
         bthreads=bthreads,
         thmode=thmode,
         http=http,
+        wsmode=wsmode,
     )
     if exc_prefix:
         proc_cmd = f'{exc_prefix}/{proc_cmd}'
@@ -113,6 +117,44 @@ def wrk(duration, concurrency, endpoint, post=False, h2=False):
         }
 
 
+def wsb(concurrency, msgs):
+    exc_prefix = os.environ.get('BENCHMARK_EXC_PREFIX')
+    cmd_parts = [
+        f'{exc_prefix}/python' if exc_prefix else 'python',
+        os.path.join(os.path.dirname(__file__), 'ws/benchmark.py'),
+    ]
+    env = dict(os.environ)
+    try:
+        proc = subprocess.run(  # noqa: S602
+            ' '.join(cmd_parts),
+            shell=True,
+            check=True,
+            capture_output=True,
+            env={
+                'BENCHMARK_CONCURRENCY': str(concurrency),
+                'BENCHMARK_MSGNO': str(msgs),
+                **env
+            }
+        )
+        return json.loads(proc.stdout.decode('utf8'))
+    except Exception as e:
+        print(f'WARN: got exception {e} while loading wsbench data')
+        return {
+            'timings': {
+                'recv': {'avg': 0, 'max': 0, 'min': 0},
+                'send': {'avg': 0, 'max': 0, 'min': 0},
+                'sum': {'avg': 0, 'max': 0, 'min': 0},
+                'all': {'avg': 0, 'max': 0, 'min': 0},
+            },
+            'throughput': {
+                'recv': 0,
+                'send': 0,
+                'all': 0,
+                'sum': 0,
+            },
+        }
+
+
 def benchmark(endpoint, post=False, h2=False, concurrencies=None):
     concurrencies = concurrencies or WRK_CONCURRENCIES
     results = {}
@@ -131,6 +173,17 @@ def benchmark(endpoint, post=False, h2=False, concurrencies=None):
     return results
 
 
+def benchmark_ws(concurrencies=None):
+    concurrencies = concurrencies or WS_CONCURRENCIES
+    results = {}
+    # bench
+    for concurrency, msgs in concurrencies:
+        res = wsb(concurrency, msgs)
+        results[concurrency] = res
+        time.sleep(2)
+    return results
+
+
 def concurrencies():
     nperm = sorted({1, 2, 4, round(CPU / 2), CPU})
     results = {'wsgi': {}}
@@ -264,6 +317,21 @@ def vs_io():
     return results
 
 
+def vs_ws():
+    results = {}
+    for fw in [
+        'granian_rsgi',
+        'granian_asgi',
+        'uvicorn_h11',
+        'hypercorn',
+    ]:
+        fw_app = fw.split('_')[1] if fw.startswith('granian') else fw
+        title = ' '.join(item.title() for item in fw.split('_'))
+        with app(fw_app, ws=True, app_path='ws.app'):
+            results[title] = benchmark_ws()
+    return results
+
+
 def _granian_version():
     import granian
 
@@ -282,6 +350,7 @@ def run():
         'vs_http2': vs_http2,
         'vs_files': vs_files,
         'vs_io': vs_io,
+        'vs_ws': vs_ws,
     }
     inp_benchmarks = sys.argv[1:] or ['base']
     if 'base' in inp_benchmarks:

diff --git a/benchmarks/pyver.md b/benchmarks/pyver.md
@@ -0,0 +1,53 @@
+# Granian benchmarks
+
+
+
+## Python versions
+
+Run at: Mon 28 Oct 2024, 02:17    
+Environment: GHA Linux x86_64 (CPUs: 4)    
+Granian version: 1.6.2    
+
+Comparison between different Python versions of Granian application protocols using 4bytes plain text response.    
+Bytes and string response are reported for every protocol just to report the difference with RSGI protocol.    
+ASGI and WSGI responses are always returned as bytes by the application.    
+The "echo" request is a 4bytes POST request responding with the same body.
+
+| Python version | Request | Total requests | RPS | avg latency | max latency |
+| --- | --- | --- | --- | --- | --- |
+| 3.10 | RSGI bytes (c64) | 445347 | 44540 | 1.433ms | 4.16ms |
+| 3.10 | RSGI str (c64) | 407154 | 40713 | 1.569ms | 4.146ms |
+| 3.10 | RSGI echo (c64) | 409490 | 40952 | 1.559ms | 3.98ms |
+| 3.10 | ASGI bytes (c64) | 404741 | 40475 | 1.577ms | 3.853ms |
+| 3.10 | ASGI str (c64) | 443742 | 44379 | 1.439ms | 3.743ms |
+| 3.10 | ASGI echo (c64) | 288973 | 28904 | 2.208ms | 4.757ms |
+| 3.10 | WSGI bytes (c128) | 559148 | 55978 | 2.28ms | 24.538ms |
+| 3.10 | WSGI str (c256) | 567857 | 56935 | 4.476ms | 66.155ms |
+| 3.10 | WSGI echo (c512) | 501010 | 50392 | 10.098ms | 117.374ms |
+| 3.11 | RSGI bytes (c64) | 408906 | 40897 | 1.56ms | 3.754ms |
+| 3.11 | RSGI str (c64) | 444267 | 44427 | 1.437ms | 4.131ms |
+| 3.11 | RSGI echo (c256) | 375214 | 37628 | 6.77ms | 69.911ms |
+| 3.11 | ASGI bytes (c128) | 394648 | 39496 | 3.232ms | 19.781ms |
+| 3.11 | ASGI str (c512) | 416410 | 41848 | 12.182ms | 104.538ms |
+| 3.11 | ASGI echo (c64) | 220900 | 22093 | 2.888ms | 5.823ms |
+| 3.11 | WSGI bytes (c128) | 381549 | 38197 | 3.339ms | 24.674ms |
+| 3.11 | WSGI str (c128) | 377113 | 37754 | 3.377ms | 32.506ms |
+| 3.11 | WSGI echo (c256) | 342484 | 34363 | 7.42ms | 66.93ms |
+| 3.12 | RSGI bytes (c512) | 423560 | 42590 | 11.958ms | 141.381ms |
+| 3.12 | RSGI str (c512) | 427772 | 43035 | 11.842ms | 114.818ms |
+| 3.12 | RSGI echo (c512) | 361753 | 36365 | 13.995ms | 169.242ms |
+| 3.12 | ASGI bytes (c128) | 419544 | 42014 | 3.032ms | 36.038ms |
+| 3.12 | ASGI str (c64) | 415682 | 41567 | 1.536ms | 3.97ms |
+| 3.12 | ASGI echo (c64) | 212445 | 21248 | 3.004ms | 6.795ms |
+| 3.12 | WSGI bytes (c256) | 356792 | 35798 | 7.121ms | 64.292ms |
+| 3.12 | WSGI str (c128) | 362003 | 36244 | 3.52ms | 20.129ms |
+| 3.12 | WSGI echo (c512) | 322632 | 32432 | 15.676ms | 144.224ms |
+| 3.13 | RSGI bytes (c512) | 400784 | 40264 | 12.635ms | 175.762ms |
+| 3.13 | RSGI str (c64) | 407448 | 40745 | 1.568ms | 4.455ms |
+| 3.13 | RSGI echo (c128) | 384481 | 38485 | 3.315ms | 23.927ms |
+| 3.13 | ASGI bytes (c512) | 414131 | 41618 | 12.238ms | 147.581ms |
+| 3.13 | ASGI str (c64) | 449520 | 44963 | 1.419ms | 3.919ms |
+| 3.13 | ASGI echo (c128) | 225465 | 22567 | 5.648ms | 36.339ms |
+| 3.13 | WSGI bytes (c128) | 371324 | 37194 | 3.429ms | 18.313ms |
+| 3.13 | WSGI str (c64) | 369008 | 36910 | 1.729ms | 5.06ms |
+| 3.13 | WSGI echo (c128) | 363711 | 36408 | 3.505ms | 20.864ms |
diff --git a/benchmarks/templates/_helpers.tpl b/benchmarks/templates/_helpers.tpl
@@ -1,6 +1,6 @@
 {{
-    def get_max_concurrency_run(data):
-        concurrency_values = {data[ckey]["requests"]["rps"]: ckey for ckey in data.keys()}
+    def get_max_concurrency_run(data, k1="requests", k2="rps"):
+        concurrency_values = {data[ckey][k1][k2]: ckey for ckey in data.keys()}
         maxc = concurrency_values[max(concurrency_values.keys())]
         return maxc, data[maxc]
 

diff --git a/benchmarks/templates/_vs_ws_table.tpl b/benchmarks/templates/_vs_ws_table.tpl
@@ -0,0 +1,8 @@
+| Server | Send throughput | Receive throughput | Combined throughput |
+| --- | --- | --- | --- |
+{{ for key, runs in _data.items(): }}
+{{ max_c, run = get_max_concurrency_run(runs, "throughput", "sum") }}
+{{ if run["throughput"]["sum"]: }}
+| {{ =key }} (c{{ =max_c }}) | {{ =round(run["throughput"]["send"]) }} | {{ =round(run["throughput"]["recv"]) }} | {{ =round(run["throughput"]["sum"]) }} |
+{{ pass }}
+{{ pass }}