[ty] Add more benchmarks

.github/actionlint.yaml

@@ -9,3 +9,4 @@ self-hosted-runner:
     - depot-ubuntu-22.04-32
     - github-windows-2025-x86_64-8
     - github-windows-2025-x86_64-16
+    - codspeed-macro

.github/workflows/ci.yaml

@@ -893,7 +893,7 @@ jobs:
         run: npm run fmt:check
         working-directory: playground
 
-  benchmarks:
+  benchmarks-instrumented:
     runs-on: ubuntu-24.04
     needs: determine_changes
     if: ${{ github.repository == 'astral-sh/ruff' && !contains(github.event.pull_request.labels.*.name, 'no-test') && (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }}
@@ -905,6 +905,40 @@ jobs:
           persist-credentials: false
 
       - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
+
+      - name: "Install Rust toolchain"
+        run: rustup show
+
+      - name: "Install codspeed"
+        uses: taiki-e/install-action@735e5933943122c5ac182670a935f54a949265c1 # v2.52.4
+        with:
+          tool: cargo-codspeed
+
+      - name: "Build benchmarks"
+        run: cargo codspeed build --features "codspeed,instrumented" --no-default-features -p ruff_benchmark
+
+      - name: "Run benchmarks"
+        uses: CodSpeedHQ/action@0010eb0ca6e89b80c88e8edaaa07cfe5f3e6664d # v3.5.0
+        with:
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}
+
+  benchmarks-walltime:
+    runs-on: codspeed-macro
+    needs: determine_changes
+    if: ${{ github.repository == 'astral-sh/ruff' && !contains(github.event.pull_request.labels.*.name, 'no-test') && (needs.determine_changes.outputs.ty == 'true' || github.ref == 'refs/heads/main') }}
+    timeout-minutes: 20
+    env:
+      TY_LOG: ruff_benchmark=debug
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
 
       - name: "Install Rust toolchain"
         run: rustup show
@@ -915,7 +949,7 @@ jobs:
           tool: cargo-codspeed
 
       - name: "Build benchmarks"
-        run: cargo codspeed build --features codspeed -p ruff_benchmark
+        run: cargo codspeed build --features "codspeed,walltime" --no-default-features -p ruff_benchmark
 
       - name: "Run benchmarks"
         uses: CodSpeedHQ/action@0010eb0ca6e89b80c88e8edaaa07cfe5f3e6664d # v3.5.0

Cargo.toml

@@ -62,6 +62,7 @@ camino = { version = "1.1.7" }
 clap = { version = "4.5.3", features = ["derive"] }
 clap_complete_command = { version = "0.6.0" }
 clearscreen = { version = "4.0.0" }
+divan = { package = "codspeed-divan-compat", version = "2.10.1" }
 codspeed-criterion-compat = { version = "2.6.0", default-features = false }
 colored = { version = "3.0.0" }
 console_error_panic_hook = { version = "0.1.7" }

crates/ruff_benchmark/Cargo.toml

@@ -19,43 +19,69 @@ doctest = false
 [[bench]]
 name = "linter"
 harness = false
+required-features = ["instrumented"]
 
 [[bench]]
 name = "lexer"
 harness = false
+required-features = ["instrumented"]
 
 [[bench]]
 name = "parser"
 harness = false
+required-features = ["instrumented"]
 
 [[bench]]
 name = "formatter"
 harness = false
+required-features = ["instrumented"]
 
 [[bench]]
 name = "ty"
 harness = false
+required-features = ["instrumented"]
+
+[[bench]]
+name = "ty_walltime"
+harness = false
+required-features = ["walltime"]
 
 [dependencies]
+ruff_db = { workspace = true, features = ["testing"] }
+ruff_python_ast = { workspace = true }
+ruff_linter = { workspace = true, optional = true }
+ruff_python_formatter = { workspace = true, optional = true }
+ruff_python_parser = { workspace = true, optional = true }
+ruff_python_trivia = { workspace = true, optional = true }
+ty_project = { workspace = true, optional = true }
+
+divan = { workspace = true, optional = true }
+anyhow = { workspace = true }
 codspeed-criterion-compat = { workspace = true, default-features = false, optional = true }
-criterion = { workspace = true, default-features = false }
+criterion = { workspace = true, default-features = false, optional = true }
 rayon = { workspace = true }
 rustc-hash = { workspace = true }
-
-[dev-dependencies]
-ruff_db = { workspace = true }
-ruff_linter = { workspace = true }
-ruff_python_ast = { workspace = true }
-ruff_python_formatter = { workspace = true }
-ruff_python_parser = { workspace = true }
-ruff_python_trivia = { workspace = true }
-ty_project = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tracing = { workspace = true }
 
 [lints]
 workspace = true
 
 [features]
+default = ["instrumented", "walltime"]
+# Enables the benchmark that should only run with codspeed's instrumented runner
+instrumented = [
+    "criterion",
+    "ruff_linter",
+    "ruff_python_formatter",
+    "ruff_python_parser",
+    "ruff_python_trivia",
+    "ty_project",
+]
 codspeed = ["codspeed-criterion-compat"]
+# Enables benchmark that should only run with codspeed's walltime runner.
+walltime = ["ruff_db/os", "ty_project", "divan"]
 
 [target.'cfg(target_os = "windows")'.dev-dependencies]
 mimalloc = { workspace = true }