Issue #4 Run only one benchmark per GitHub workflow trigger

just a random selection for now

.github/workflows/benchmarks.yaml

@@ -1,7 +1,6 @@
 name: "Benchmarks"
 
 # TODO: trigger based on schedule
-# TODO: only run subset of benchmarks
 on: [push]
 
 jobs:
@@ -20,4 +19,4 @@ jobs:
       - name: Test with pytest
         run: |
           cd qa/benchmarks
-          pytest
+          pytest --random-subset=1

qa/benchmarks/tests/conftest.py

@@ -0,0 +1,31 @@
+import logging
+import random
+
+# TODO: how to make sure the logging/printing from this plugin is actually visible by default?
+_log = logging.getLogger(__name__)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--random-subset",
+        metavar="N",
+        action="store",
+        default=-1,
+        type=int,
+        help="Only run random selected subset benchmarks.",
+    )
+
+
+def pytest_collection_modifyitems(session, config, items):
+    """
+    Pytest plugin to select a random subset of benchmarks to run.
+
+    based on https://alexwlchan.net/til/2024/run-random-subset-of-tests-in-pytest/
+    """
+    subset_size = config.getoption("--random-subset")
+
+    if subset_size >= 0:
+        _log.warning(
+            f"Selecting random subset of {subset_size} from {len(items)} benchmarks."
+        )
+        items[:] = random.sample(items, k=subset_size)