Auto merge of rust-lang#136585 - gvozdvmozgu:memchr-eat-until-lexer, r=lcnr

implement `eat_until` leveraging memchr in lexer

Author: bors

Cargo.lock

@@ -4064,6 +4064,7 @@ name = "rustc_lexer"
 version = "0.0.0"
 dependencies = [
  "expect-test",
+ "memchr",
  "unicode-properties",
  "unicode-xid",
 ]

compiler/rustc_lexer/Cargo.toml

@@ -14,6 +14,7 @@ Rust lexer used by rustc. No stability guarantees are provided.
 
 # Note that this crate purposefully does not depend on other rustc crates
 [dependencies]
+memchr = "2.7.4"
 unicode-xid = "0.2.0"
 
 [dependencies.unicode-properties]

compiler/rustc_lexer/src/cursor.rs

@@ -103,4 +103,11 @@ impl<'a> Cursor<'a> {
             self.bump();
         }
     }
+
+    pub(crate) fn eat_until(&mut self, byte: u8) {
+        self.chars = match memchr::memchr(byte, self.as_str().as_bytes()) {
+            Some(index) => self.as_str()[index..].chars(),
+            None => "".chars(),
+        }
+    }
 }

compiler/rustc_lexer/src/lib.rs

@@ -483,7 +483,7 @@ impl Cursor<'_> {
             _ => None,
         };
 
-        self.eat_while(|c| c != '\n');
+        self.eat_until(b'\n');
         LineComment { doc_style }
     }
 
@@ -888,7 +888,7 @@ impl Cursor<'_> {
         // Skip the string contents and on each '#' character met, check if this is
         // a raw string termination.
         loop {
-            self.eat_while(|c| c != '"');
+            self.eat_until(b'"');
 
             if self.is_eof() {
                 return Err(RawStrError::NoTerminator {