ci(benchmarks/lexer): ensure next_token is inlined into lexer benchmark (#15519)

Preparatory step for #15513. That PR was showing a massive slowdown on lexer benchmarks, but it was only due to the change in that PR resulting in `next_token` not being inlined into the lexer benchmark.

Add a separate function `next_token_for_benchmarks` which has identical context as `next_token`, but is marked `#[inline(always)]`, and use it in lexer benchmark instead. This fixes the problem with the benchmark in #15513.

Author: overlookmotel

crates/oxc_parser/src/lexer/mod.rs

@@ -214,6 +214,16 @@ impl<'a> Lexer<'a> {
         self.finish_next(kind)
     }
 
+    // This is a workaround for a problem where `next_token` is not inlined in lexer benchmark.
+    // Must be kept in sync with `next_token` above, and contain exactly the same code.
+    #[cfg(feature = "benchmarking")]
+    #[expect(clippy::inline_always)]
+    #[inline(always)]
+    pub fn next_token_for_benchmarks(&mut self) -> Token {
+        let kind = self.read_next_token();
+        self.finish_next(kind)
+    }
+
     fn finish_next(&mut self, kind: Kind) -> Token {
         self.token.set_kind(kind);
         self.token.set_end(self.offset());

tasks/benchmark/benches/lexer.rs

@@ -73,7 +73,10 @@ fn lex_whole_file<'a>(
 ) -> Lexer<'a> {
     let mut lexer = Lexer::new_for_benchmarks(allocator, source_text, source_type);
     if lexer.first_token().kind() != Kind::Eof {
-        while lexer.next_token().kind() != Kind::Eof {}
+        // Use `next_token_for_benchmarks` instead of `next_token`, to work around problem
+        // where `next_token` wasn't inlined here.
+        // `next_token_for_benchmarks` is identical to `next_token`, but is marked `#[inline(always)]`.
+        while lexer.next_token_for_benchmarks().kind() != Kind::Eof {}
     }
     lexer
 }