[syntax-errors] PEP 701 f-strings before Python 3.12

crates/ruff_python_formatter/resources/test/fixtures/black/cases/fstring.options.json

@@ -0,0 +1 @@
+{"target_version": "3.12"}

crates/ruff_python_parser/resources/inline/err/pep701_f_string_py311.py

@@ -0,0 +1,12 @@
+# parse_options: {"target-version": "3.11"}
+f'Magic wand: { bag['wand'] }'     # nested quotes
+f"{'\n'.join(a)}"                  # escape sequence
+f'''A complex trick: {
+    bag['bag']                     # comment
+}'''
+f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"  # arbitrary nesting
+f"{f'''{"nested"} inner'''} outer" # nested (triple) quotes
+f"test {a \
+    } more"                        # line continuation
+f"""{f"""{x}"""}"""                # mark the whole triple quote
+f"{'\n'.join(['\t', '\v', '\r'])}"  # multiple escape sequences, multiple errors

crates/ruff_python_parser/resources/inline/ok/pep701_f_string_py311.py

@@ -0,0 +1,7 @@
+# parse_options: {"target-version": "3.11"}
+f"outer {'# not a comment'}"
+f'outer {x:{"# not a comment"} }'
+f"""{f'''{f'{"# not a comment"}'}'''}"""
+f"""{f'''# before expression {f'# aro{f"#{1+1}#"}und #'}'''} # after expression"""
+f"escape outside of \t {expr}\n"
+f"test\"abcd"

crates/ruff_python_parser/resources/inline/ok/pep701_f_string_py312.py

@@ -0,0 +1,10 @@
+# parse_options: {"target-version": "3.12"}
+f'Magic wand: { bag['wand'] }'     # nested quotes
+f"{'\n'.join(a)}"                  # escape sequence
+f'''A complex trick: {
+    bag['bag']                     # comment
+}'''
+f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"  # arbitrary nesting
+f"{f'''{"nested"} inner'''} outer" # nested (triple) quotes
+f"test {a \
+    } more"                        # line continuation

crates/ruff_python_parser/src/error.rs

@@ -452,6 +452,14 @@ pub enum StarTupleKind {
     Yield,
 }
 
+/// The type of PEP 701 f-string error for [`UnsupportedSyntaxErrorKind::Pep701FString`].
+#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
+pub enum FStringKind {
+    Backslash,
+    Comment,
+    NestedQuote,
+}
+
 #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
 pub enum UnparenthesizedNamedExprKind {
     SequenceIndex,
@@ -661,6 +669,34 @@ pub enum UnsupportedSyntaxErrorKind {
     TypeAliasStatement,
     TypeParamDefault,
 
+    /// Represents the use of a [PEP 701] f-string before Python 3.12.
+    ///
+    /// ## Examples
+    ///
+    /// As described in the PEP, each of these cases were invalid before Python 3.12:
+    ///
+    /// ```python
+    /// # nested quotes
+    /// f'Magic wand: { bag['wand'] }'
+    ///
+    /// # escape characters
+    /// f"{'\n'.join(a)}"
+    ///
+    /// # comments
+    /// f'''A complex trick: {
+    ///     bag['bag']  # recursive bags!
+    /// }'''
+    ///
+    /// # arbitrary nesting
+    /// f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"
+    /// ```
+    ///
+    /// These restrictions were lifted in Python 3.12, meaning that all of these examples are now
+    /// valid.
+    ///
+    /// [PEP 701]: https://peps.python.org/pep-0701/
+    Pep701FString(FStringKind),
+
     /// Represents the use of a parenthesized `with` item before Python 3.9.
     ///
     /// ## Examples
@@ -838,6 +874,15 @@ impl Display for UnsupportedSyntaxError {
             UnsupportedSyntaxErrorKind::TypeParamDefault => {
                 "Cannot set default type for a type parameter"
             }
+            UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::Backslash) => {
+                "Cannot use an escape sequence (backslash) in f-strings"
+            }
+            UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::Comment) => {
+                "Cannot use comments in f-strings"
+            }
+            UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::NestedQuote) => {
+                "Cannot reuse outer quote character in f-strings"
+            }
             UnsupportedSyntaxErrorKind::ParenthesizedContextManager => {
                 "Cannot use parentheses within a `with` statement"
             }
@@ -904,6 +949,7 @@ impl UnsupportedSyntaxErrorKind {
             UnsupportedSyntaxErrorKind::TypeParameterList => Change::Added(PythonVersion::PY312),
             UnsupportedSyntaxErrorKind::TypeAliasStatement => Change::Added(PythonVersion::PY312),
             UnsupportedSyntaxErrorKind::TypeParamDefault => Change::Added(PythonVersion::PY313),
+            UnsupportedSyntaxErrorKind::Pep701FString(_) => Change::Added(PythonVersion::PY312),
             UnsupportedSyntaxErrorKind::ParenthesizedContextManager => {
                 Change::Added(PythonVersion::PY39)
             }

crates/ruff_python_parser/src/parser/expression.rs

@@ -11,13 +11,15 @@ use ruff_python_ast::{
 };
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 
-use crate::error::{StarTupleKind, UnparenthesizedNamedExprKind};
+use crate::error::{FStringKind, StarTupleKind, UnparenthesizedNamedExprKind};
 use crate::parser::progress::ParserProgress;
 use crate::parser::{helpers, FunctionKind, Parser};
 use crate::string::{parse_fstring_literal_element, parse_string_literal, StringType};
 use crate::token::{TokenKind, TokenValue};
 use crate::token_set::TokenSet;
-use crate::{FStringErrorType, Mode, ParseErrorType, UnsupportedSyntaxErrorKind};
+use crate::{
+    FStringErrorType, Mode, ParseErrorType, UnsupportedSyntaxError, UnsupportedSyntaxErrorKind,
+};
 
 use super::{FStringElementsKind, Parenthesized, RecoveryContextKind};
 
@@ -1393,13 +1395,89 @@ impl<'src> Parser<'src> {
 
         self.expect(TokenKind::FStringEnd);
 
+        // test_ok pep701_f_string_py312
+        // # parse_options: {"target-version": "3.12"}
+        // f'Magic wand: { bag['wand'] }'     # nested quotes
+        // f"{'\n'.join(a)}"                  # escape sequence
+        // f'''A complex trick: {
+        //     bag['bag']                     # comment
+        // }'''
+        // f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"  # arbitrary nesting
+        // f"{f'''{"nested"} inner'''} outer" # nested (triple) quotes
+        // f"test {a \
+        //     } more"                        # line continuation
+
+        // test_ok pep701_f_string_py311
+        // # parse_options: {"target-version": "3.11"}
+        // f"outer {'# not a comment'}"
+        // f'outer {x:{"# not a comment"} }'
+        // f"""{f'''{f'{"# not a comment"}'}'''}"""
+        // f"""{f'''# before expression {f'# aro{f"#{1+1}#"}und #'}'''} # after expression"""
+        // f"escape outside of \t {expr}\n"
+        // f"test\"abcd"
+
+        // test_err pep701_f_string_py311
+        // # parse_options: {"target-version": "3.11"}
+        // f'Magic wand: { bag['wand'] }'     # nested quotes
+        // f"{'\n'.join(a)}"                  # escape sequence
+        // f'''A complex trick: {
+        //     bag['bag']                     # comment
+        // }'''
+        // f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"  # arbitrary nesting
+        // f"{f'''{"nested"} inner'''} outer" # nested (triple) quotes
+        // f"test {a \
+        //     } more"                        # line continuation
+        // f"""{f"""{x}"""}"""                # mark the whole triple quote
+        // f"{'\n'.join(['\t', '\v', '\r'])}"  # multiple escape sequences, multiple errors
+
+        let range = self.node_range(start);
+
+        if !self.options.target_version.supports_pep_701() {
+            let quote_bytes = flags.quote_str().as_bytes();
+            let quote_len = flags.quote_len();
+            for expr in elements.expressions() {
+                for slash_position in memchr::memchr_iter(b'\\', self.source[expr.range].as_bytes())
+                {
+                    let slash_position = TextSize::try_from(slash_position).unwrap();
+                    self.add_unsupported_syntax_error(
+                        UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::Backslash),
+                        TextRange::at(expr.range.start() + slash_position, '\\'.text_len()),
+                    );
+                }
+
+                if let Some(quote_position) =
+                    memchr::memmem::find(self.source[expr.range].as_bytes(), quote_bytes)
+                {
+                    let quote_position = TextSize::try_from(quote_position).unwrap();
+                    self.add_unsupported_syntax_error(
+                        UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::NestedQuote),
+                        TextRange::at(expr.range.start() + quote_position, quote_len),
+                    );
+                };
+            }
+
+            self.check_fstring_comments(range);
+        }
+
         ast::FString {
             elements,
-            range: self.node_range(start),
+            range,
             flags: ast::FStringFlags::from(flags),
         }
     }
 
+    /// Check `range` for comment tokens and report an `UnsupportedSyntaxError` for each one found.
+    fn check_fstring_comments(&mut self, range: TextRange) {
+        self.unsupported_syntax_errors
+            .extend(self.tokens.in_range(range).iter().filter_map(|token| {
+                token.kind().is_comment().then_some(UnsupportedSyntaxError {
+                    kind: UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::Comment),
+                    range: token.range(),
+                    target_version: self.options.target_version,
+                })
+            }));
+    }
+
     /// Parses a list of f-string elements.
     ///
     /// # Panics

crates/ruff_python_parser/src/token.rs

@@ -418,6 +418,12 @@ impl TokenKind {
         matches!(self, TokenKind::Comment | TokenKind::NonLogicalNewline)
     }
 
+    /// Returns `true` if this is a comment token.
+    #[inline]
+    pub const fn is_comment(&self) -> bool {
+        matches!(self, TokenKind::Comment)
+    }
+
     #[inline]
     pub const fn is_arithmetic(self) -> bool {
         matches!(

crates/ruff_python_parser/src/token_source.rs

@@ -166,6 +166,21 @@ impl<'src> TokenSource<'src> {
         self.tokens.truncate(tokens_position);
     }
 
+    /// Returns a slice of [`Token`] that are within the given `range`.
+    pub(crate) fn in_range(&self, range: TextRange) -> &[Token] {
+        let start = self
+            .tokens
+            .iter()
+            .rposition(|tok| tok.start() == range.start());
+        let end = self.tokens.iter().rposition(|tok| tok.end() == range.end());
+
+        let (Some(start), Some(end)) = (start, end) else {
+            return &self.tokens;
+        };
+
+        &self.tokens[start..=end]
+    }
+
     /// Consumes the token source, returning the collected tokens, comment ranges, and any errors
     /// encountered during lexing. The token collection includes both the trivia and non-trivia
     /// tokens.