docs(parser): document is_on_new_line and read_bool performance characteristics

crates/oxc_parser/src/lexer/token.rs

@@ -130,6 +130,11 @@ impl Token {
         self.0 |= u128::from(kind as u8) << KIND_SHIFT;
     }
 
+    /// Checks if this token appears at the start of a new line.
+    ///
+    /// Returns `true` if the token was preceded by a line terminator during lexical analysis.
+    /// This information is crucial for automatic semicolon insertion (ASI) and other
+    /// JavaScript parsing rules that depend on line boundaries.
     #[inline]
     pub fn is_on_new_line(&self) -> bool {
         // Use a pointer read rather than arithmetic as it produces less instructions.
@@ -188,9 +193,35 @@ impl Token {
 
     /// Read `bool` from 8 bits starting at bit position `shift`.
     ///
+    /// This method uses unsafe pointer arithmetic to directly read a boolean value
+    /// from the token's 128-bit representation. This approach is deliberately chosen
+    /// for performance optimization on hot paths.
+    ///
+    /// # Performance Analysis
+    ///
+    /// This unsafe pointer arithmetic approach generates only 3 CPU instructions:
+    /// ```asm
+    /// movzx   eax, BYTE PTR [rdi+9]  ; Load byte at offset
+    /// and     eax, 1                  ; Mask to get boolean
+    /// ret                             ; Return
+    /// ```
+    ///
+    /// Compared to the safe bit-shift alternative which generates 4 instructions:
+    /// ```asm
+    /// mov     rax, QWORD PTR [rdi+8]  ; Load 64-bit value
+    /// shr     rax, 8                  ; Shift to extract bit
+    /// and     eax, 1                  ; Mask to get boolean
+    /// ret                             ; Return
+    /// ```
+    ///
+    /// This optimization was retained after careful benchmarking (see PR #13788),
+    /// where the single instruction difference on hot paths justified keeping
+    /// the unsafe implementation.
+    ///
     /// # SAFETY
     /// `shift` must be the location of a valid boolean "field" in [`Token`]
-    /// e.g. `ESCAPED_SHIFT`
+    /// e.g. `ESCAPED_SHIFT`. The caller must guarantee that the 8 bits at
+    /// `shift` contain only 0 or 1, making it safe to read as a `bool`.
     #[expect(clippy::inline_always)]
     #[inline(always)] // So `shift` is statically known
     unsafe fn read_bool(&self, shift: usize) -> bool {