Skip to content

Commit 4a79328

Browse files
[3.9] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) (GH-107224) (#107231)
Previously *consumed was not set in this case. (cherry picked from commit f08e52c). (cherry picked from commit b8b3e6a)
1 parent 264b1da commit 4a79328

File tree

4 files changed

+88
-1
lines changed

4 files changed

+88
-1
lines changed

Lib/test/test_unicode.py

+47
Original file line numberDiff line numberDiff line change
@@ -2870,6 +2870,53 @@ def test_asutf8andsize(self):
28702870
self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4))
28712871
self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc')
28722872

2873+
@support.cpython_only
2874+
def test_decodeutf8(self):
2875+
"""Test PyUnicode_DecodeUTF8()"""
2876+
import _testcapi
2877+
decodeutf8 = _testcapi.unicode_decodeutf8
2878+
2879+
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
2880+
b = s.encode('utf-8')
2881+
self.assertEqual(decodeutf8(b), s)
2882+
self.assertEqual(decodeutf8(b, 'strict'), s)
2883+
2884+
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80')
2885+
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0')
2886+
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff')
2887+
self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f')
2888+
self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd')
2889+
self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb')
2890+
2891+
self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo')
2892+
# TODO: Test PyUnicode_DecodeUTF8() with NULL as data and
2893+
# negative size.
2894+
2895+
@support.cpython_only
2896+
def test_decodeutf8stateful(self):
2897+
"""Test PyUnicode_DecodeUTF8Stateful()"""
2898+
import _testcapi
2899+
decodeutf8stateful = _testcapi.unicode_decodeutf8stateful
2900+
2901+
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
2902+
b = s.encode('utf-8')
2903+
self.assertEqual(decodeutf8stateful(b), (s, len(b)))
2904+
self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b)))
2905+
2906+
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80')
2907+
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0')
2908+
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff')
2909+
self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1))
2910+
self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1))
2911+
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb')
2912+
self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4))
2913+
2914+
self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo')
2915+
# TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and
2916+
# negative size.
2917+
# TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of
2918+
# "consumed".
2919+
28732920
# Test PyUnicode_FindChar()
28742921
@support.cpython_only
28752922
def test_findchar(self):
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data:
2+
``*consumed`` was not set.

Modules/_testcapimodule.c

+36-1
Original file line numberDiff line numberDiff line change
@@ -1987,6 +1987,40 @@ unicode_asutf8andsize(PyObject *self, PyObject *args)
19871987
return Py_BuildValue("(Nn)", result, utf8_len);
19881988
}
19891989

1990+
/* Test PyUnicode_DecodeUTF8() */
1991+
static PyObject *
1992+
unicode_decodeutf8(PyObject *self, PyObject *args)
1993+
{
1994+
const char *data;
1995+
Py_ssize_t size;
1996+
const char *errors = NULL;
1997+
1998+
if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors))
1999+
return NULL;
2000+
2001+
return PyUnicode_DecodeUTF8(data, size, errors);
2002+
}
2003+
2004+
/* Test PyUnicode_DecodeUTF8Stateful() */
2005+
static PyObject *
2006+
unicode_decodeutf8stateful(PyObject *self, PyObject *args)
2007+
{
2008+
const char *data;
2009+
Py_ssize_t size;
2010+
const char *errors = NULL;
2011+
Py_ssize_t consumed = 123456789;
2012+
PyObject *result;
2013+
2014+
if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors))
2015+
return NULL;
2016+
2017+
result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed);
2018+
if (!result) {
2019+
return NULL;
2020+
}
2021+
return Py_BuildValue("(Nn)", result, consumed);
2022+
}
2023+
19902024
static PyObject *
19912025
unicode_findchar(PyObject *self, PyObject *args)
19922026
{
@@ -5502,7 +5536,8 @@ static PyMethodDef TestMethods[] = {
55025536
{"unicode_asucs4", unicode_asucs4, METH_VARARGS},
55035537
{"unicode_asutf8", unicode_asutf8, METH_VARARGS},
55045538
{"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS},
5505-
{"unicode_findchar", unicode_findchar, METH_VARARGS},
5539+
{"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS},
5540+
{"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, {"unicode_findchar", unicode_findchar, METH_VARARGS},
55065541
{"unicode_copycharacters", unicode_copycharacters, METH_VARARGS},
55075542
{"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS},
55085543
{"unicode_transformdecimaltoascii", unicode_transformdecimaltoascii, METH_VARARGS},

Objects/unicodeobject.c

+3
Original file line numberDiff line numberDiff line change
@@ -5052,6 +5052,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size,
50525052
}
50535053
s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u));
50545054
if (s == end) {
5055+
if (consumed) {
5056+
*consumed = size;
5057+
}
50555058
return u;
50565059
}
50575060

0 commit comments

Comments
 (0)