Fix behavior of tag'\N{BLAH}'

This should treat `{BLAH}` as an interpolation. To fix this we modify `_PyPegen_joined_str()` to take an `is_raw` flag.
python · Apr 26, 2023 · f93052d · f93052d
1 parent c1c75ea
commit f93052d
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 6 deletions.
diff --git a/Grammar/python.gram b/Grammar/python.gram
@@ -894,7 +894,7 @@ fstring_format_spec[expr_ty]:
     | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
     | fstring_replacement_field
 fstring[expr_ty]:
-    | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, a, (asdl_expr_seq*)b, c) }
+    | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, 0, a, (asdl_expr_seq*)b, c) }
 tagstring[expr_ty] (memo):
     | a=TAGSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_tag_str(p, a, (asdl_expr_seq*)b, c) }
 

diff --git a/Lib/test/test_tag_strings.py b/Lib/test/test_tag_strings.py
@@ -61,6 +61,19 @@ def tag(*args):
         self.assertEqual(conv, "r")
         self.assertEqual(spec, None)
 
+    def test_unicode_names(self):
+        def tag(*args):
+            return args
+        res = tag"\N{SPACE}"
+        self.assertEqual(len(res), 2)
+        self.assertEqual(res[0], r"\N")
+        func, string, conv, spec = res[1]
+        SPACE = 42
+        self.assertEqual(func(), 42)
+        self.assertEqual(string, "SPACE")
+        self.assertEqual(conv, None)
+        self.assertEqual(spec, None)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
@@ -1290,15 +1290,15 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions)
 }
 
 expr_ty
-_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) {
+_PyPegen_joined_str(Parser *p, int is_raw, Token* a, asdl_expr_seq* raw_expressions, Token*b) {
     asdl_expr_seq *expr = unpack_top_level_joined_strs(p, raw_expressions);
     Py_ssize_t n_items = asdl_seq_LEN(expr);
 
     const char* quote_str = PyBytes_AsString(a->bytes);
     if (quote_str == NULL) {
         return NULL;
     }
-    int is_raw = strpbrk(quote_str, "rR") != NULL;
+    is_raw = is_raw || strpbrk(quote_str, "rR") != NULL;
 
     asdl_expr_seq *seq = _Py_asdl_expr_seq_new(n_items, p->arena);
     if (seq == NULL) {
@@ -1393,7 +1393,7 @@ _PyPegen_tag_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) {
     if (tag == NULL) {
         return NULL;
     }
-    expr_ty str = _PyPegen_joined_str(p, a, raw_expressions, b);
+    expr_ty str = _PyPegen_joined_str(p, 1, a, raw_expressions, b);
     if (str == NULL) {
         return NULL;
     }

diff --git a/Parser/parser.c b/Parser/parser.c
diff --git a/Parser/pegen.h b/Parser/pegen.h
@@ -346,7 +346,7 @@ mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompiler
 asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);
 
 // TODO: move to the correct place in this file
-expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* expr, Token*b);
+expr_ty _PyPegen_joined_str(Parser *p, int is_raw, Token* a, asdl_expr_seq* expr, Token*b);
 expr_ty _PyPegen_tag_str(Parser *p, Token* a, asdl_expr_seq* expr, Token*b);
 
 // Generated function in parse.c - function definition in python.gram