From e821e2355aca112ae8d2505924e8cc410493e71a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 14 May 2020 17:49:25 +0100 Subject: [PATCH 1/3] bpo-40619: Correctly handle error text and offset for input --- Lib/test/test_exceptions.py | 1 + Parser/pegen/pegen.c | 27 ++------------------------- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index b689ec7aed18de..aba69b7dbf8dc4 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -228,6 +228,7 @@ def bar(): def baz(): '''quux''' """, 9, 20) + check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 5) diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index 083088bd9657bd..c4bd30a011c5a2 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -300,30 +300,6 @@ raise_tokenizer_init_error(PyObject *filename) Py_XDECREF(tuple); } -static inline PyObject * -get_error_line(char *buffer, int is_file) -{ - const char *newline; - if (is_file) { - newline = strrchr(buffer, '\n'); - } else { - newline = strchr(buffer, '\n'); - } - - if (is_file) { - while (newline > buffer && newline[-1] == '\n') { - --newline; - } - } - - if (newline) { - return PyUnicode_DecodeUTF8(buffer, newline - buffer, "replace"); - } - else { - return PyUnicode_DecodeUTF8(buffer, strlen(buffer), "replace"); - } -} - static int tokenizer_error(Parser *p) { @@ -422,7 +398,8 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, } if (!error_line) { - error_line = get_error_line(p->tok->buf, p->start_rule == Py_file_input); + Py_ssize_t size = p->tok->inp - p->tok->buf; + error_line = PyUnicode_DecodeUTF8(p->tok->buf, size ? size - 1: size, "replace"); if (!error_line) { goto error; } From 3e395f06792cc327577afcf5148bdb0edee028e1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 14 May 2020 20:37:31 +0100 Subject: [PATCH 2/3] fixup! bpo-40619: Correctly handle error text and offset for input --- Lib/test/test_exceptions.py | 1 + Parser/pegen/pegen.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index aba69b7dbf8dc4..efd77fdbaabe1b 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -229,6 +229,7 @@ def baz(): '''quux''' """, 9, 20) check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) + check("(1+)", 1, 4) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 5) diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index c4bd30a011c5a2..21cf91ee1a29e7 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -399,7 +399,9 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, if (!error_line) { Py_ssize_t size = p->tok->inp - p->tok->buf; - error_line = PyUnicode_DecodeUTF8(p->tok->buf, size ? size - 1: size, "replace"); + error_line = PyUnicode_DecodeUTF8(p->tok->buf, + p->tok->buf[size-1] == '\n' ? size - 1 : size, + "replace"); if (!error_line) { goto error; } From 8a4ef9b167ec81ec8e42f773ace81d6bc024b6a2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 14 May 2020 20:43:16 +0100 Subject: [PATCH 3/3] fixup! fixup! bpo-40619: Correctly handle error text and offset for input --- Parser/pegen/pegen.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index 21cf91ee1a29e7..8b79a7364758e4 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -399,9 +399,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, if (!error_line) { Py_ssize_t size = p->tok->inp - p->tok->buf; - error_line = PyUnicode_DecodeUTF8(p->tok->buf, - p->tok->buf[size-1] == '\n' ? size - 1 : size, - "replace"); + if (size && p->tok->buf[size-1] == '\n') { + size--; + } + error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace"); if (!error_line) { goto error; }