Parse arbitrarily nested f-strings (#1026)

Author: zsol

native/libcst/src/tokenizer/core/mod.rs

@@ -907,8 +907,8 @@ impl<'t> TokState<'t> {
                     return Err(TokError::UnterminatedString);
                 }
                 (ch @ Some('\''), _) | (ch @ Some('"'), _) => {
-                    // see if this actually terminates something in fstring_stack
-                    for node in self.fstring_stack.iter() {
+                    // see if this actually terminates the most recent fstring
+                    if let Some(node) = self.fstring_stack.last() {
                         if ch == Some(node.quote_char.into()) {
                             match node.quote_size {
                                 StringQuoteSize::Single => {
@@ -1004,27 +1004,18 @@ impl<'t> TokState<'t> {
 
     fn maybe_consume_fstring_end(&mut self) -> Option<TokType> {
         let ch = self.text_pos.peek();
-        let mut match_idx = None;
-        for (idx, node) in self.fstring_stack.iter().enumerate() {
+        if let Some(node) = self.fstring_stack.last() {
             if ch == Some(node.quote_char.into()) {
                 if node.quote_size == StringQuoteSize::Triple {
-                    if self.text_pos.consume(node.quote_char.triple_str()) {
-                        match_idx = Some(idx);
-                        break;
-                    }
+                    self.text_pos.consume(node.quote_char.triple_str());
                 } else {
                     self.text_pos.next(); // already matched
-                    match_idx = Some(idx);
-                    break;
                 }
+                self.fstring_stack.pop();
+                return Some(TokType::FStringEnd);
             }
         }
-        if let Some(match_idx) = match_idx {
-            self.fstring_stack.truncate(match_idx);
-            Some(TokType::FStringEnd)
-        } else {
-            None
-        }
+        None
     }
 }
 

native/libcst/src/tokenizer/tests.rs

@@ -853,3 +853,25 @@ fn test_nested_f_string_specs() {
         ])
     )
 }
+
+#[test]
+fn test_nested_f_strings() {
+    let config = TokConfig {
+        split_fstring: true,
+        ..default_config()
+    };
+    assert_eq!(
+        tokenize_all("f'{f'{2}'}'", &config),
+        Ok(vec![
+            (TokType::FStringStart, "f'"),
+            (TokType::Op, "{"),
+            (TokType::FStringStart, "f'"),
+            (TokType::Op, "{"),
+            (TokType::Number, "2"),
+            (TokType::Op, "}"),
+            (TokType::FStringEnd, "'"),
+            (TokType::Op, "}"),
+            (TokType::FStringEnd, "'")
+        ])
+    )
+}

native/libcst/tests/fixtures/super_strings.py

@@ -31,4 +31,11 @@
 f"regexp_like(path, '.*\{file_type}$')"
 f"\lfoo"
 
-f"{_:{_:}{a}}"
+f"{_:{_:}{a}}"
+
+f"foo {f"bar {x}"} baz"
+f'some words {a+b:.3f} more words {c+d=} final words'
+f"{'':*^{1:{1}}}"
+f"{'':*^{1:{1:{1}}}}"
+f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"
+