Skip to content

Commit 363de28

Browse files
committed
fixup! fixup! fixup! Handle \r\n in continuation lines
1 parent 559487c commit 363de28

File tree

2 files changed

+13
-0
lines changed

2 files changed

+13
-0
lines changed

Lib/test/test_tokenize.py

+10
Original file line numberDiff line numberDiff line change
@@ -2104,6 +2104,10 @@ def test_string(self):
21042104
b\
21052105
c"""', """\
21062106
STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4)
2107+
""")
2108+
2109+
self.check_tokenize(r'"hola\\\r\ndfgf"', """\
2110+
STRING \'"hola\\\\\\\\\\\\r\\\\ndfgf"\' (1, 0) (1, 16)
21072111
""")
21082112

21092113
self.check_tokenize('f"abc"', """\
@@ -2140,6 +2144,12 @@ def test_string(self):
21402144
FSTRING_START 'Rf"' (1, 0) (1, 3)
21412145
FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 3) (2, 3)
21422146
FSTRING_END '"' (2, 3) (2, 4)
2147+
""")
2148+
2149+
self.check_tokenize(r'f"hola\\\r\ndfgf"', """\
2150+
FSTRING_START \'f"\' (1, 0) (1, 2)
2151+
FSTRING_MIDDLE 'hola\\\\\\\\\\\\r\\\\ndfgf' (1, 2) (1, 16)
2152+
FSTRING_END \'"\' (1, 16) (1, 17)
21432153
""")
21442154

21452155
def test_function(self):

Parser/tokenizer.c

+3
Original file line numberDiff line numberDiff line change
@@ -2707,6 +2707,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
27072707
return MAKE_TOKEN(FSTRING_MIDDLE);
27082708
} else if (c == '\\') {
27092709
int peek = tok_nextc(tok);
2710+
if (peek == '\r') {
2711+
peek = tok_nextc(tok);
2712+
}
27102713
// Special case when the backslash is right before a curly
27112714
// brace. We have to restore and return the control back
27122715
// to the loop for the next iteration.

0 commit comments

Comments
 (0)