gh-104741: Add line number attribute to indentation error exception (#…

…104743)
python · May 22, 2023 · 729b252 · 729b252
1 parent 0a77960
commit 729b252
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 8 deletions.
diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py
@@ -317,7 +317,7 @@ def test_with_errored_file(self):
         with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:
             stderr  = f"{file_path!r}: Token Error: "
             stderr += ('unindent does not match any outer indentation level'
-                    ' (<tokenize>, line 3)')
+                       ' (<string>, line 3)')
             self.validate_cmd(file_path, stderr=stderr, expect_failure=True)
 
     def test_with_error_free_file(self):

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
@@ -92,9 +92,18 @@ def k(x):
         readline = BytesIO(indent_error_file).readline
         with self.assertRaisesRegex(IndentationError,
                                     "unindent does not match any "
-                                    "outer indentation level"):
+                                    "outer indentation level") as e:
             for tok in tokenize(readline):
                 pass
+        self.assertEqual(e.exception.lineno, 3)
+        self.assertEqual(e.exception.filename, '<string>')
+        self.assertEqual(e.exception.end_lineno, None)
+        self.assertEqual(e.exception.end_offset, None)
+        self.assertEqual(
+            e.exception.msg,
+            'unindent does not match any outer indentation level')
+        self.assertEqual(e.exception.offset, 9)
+        self.assertEqual(e.exception.text, '  x += 5\n')
 
     def test_int(self):
         # Ordinary integers and binary operators

diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
@@ -89,11 +89,9 @@ _tokenizer_error(struct tok_state *tok)
             }
             return -1;
         case E_DEDENT:
-            PyErr_Format(PyExc_IndentationError,
-                        "unindent does not match any outer indentation level "
-                        "(<tokenize>, line %d)",
-                        tok->lineno);
-            return -1;
+            msg = "unindent does not match any outer indentation level";
+            errtype = PyExc_IndentationError;
+            break;
         case E_INTR:
             if (!PyErr_Occurred()) {
                 PyErr_SetNone(PyExc_KeyboardInterrupt);
@@ -131,7 +129,12 @@ _tokenizer_error(struct tok_state *tok)
         goto exit;
     }
 
-    tmp = Py_BuildValue("(OnnOii)", tok->filename, tok->lineno, 0, error_line, 0, 0);
+    Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, tok->inp - tok->buf);
+    if (offset == -1) {
+        result = -1;
+        goto exit;
+    }
+    tmp = Py_BuildValue("(OnnOOO)", tok->filename, tok->lineno, offset, error_line, Py_None, Py_None);
     if (!tmp) {
         result = -1;
         goto exit;