From 2923b49ca63c59caad1a638b2c3c0fd424f0814f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 15 Apr 2022 18:38:27 +0300 Subject: [PATCH 1/4] gh-91575: Update case-insensitive matching in re to the latest Unicode version --- Lib/re/_compiler.py | 31 +++++++++++-- Lib/test/test_re.py | 43 ++++++++++++++++--- ...2-04-15-18-38-21.gh-issue-91575.fSyAxS.rst | 2 + 3 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-04-15-18-38-21.gh-issue-91575.fSyAxS.rst diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 5b4c4a3f30a40c..715e03a647c71d 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -57,6 +57,22 @@ (0x3c2, 0x3c3), # ςσ # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL (0x3c6, 0x3d5), # φϕ + # CYRILLIC SMALL LETTER VE, CYRILLIC SMALL LETTER ROUNDED VE + (0x432, 0x1c80), # вᲀ + # CYRILLIC SMALL LETTER DE, CYRILLIC SMALL LETTER LONG-LEGGED DE + (0x434, 0x1c81), # дᲁ + # CYRILLIC SMALL LETTER O, CYRILLIC SMALL LETTER NARROW O + (0x43e, 0x1c82), # оᲂ + # CYRILLIC SMALL LETTER ES, CYRILLIC SMALL LETTER WIDE ES + (0x441, 0x1c83), # сᲃ + # CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE + (0x442, 0x1c84, 0x1c85), # тᲄᲅ + # CYRILLIC SMALL LETTER HARD SIGN, CYRILLIC SMALL LETTER TALL HARD SIGN + (0x44a, 0x1c86), # ъᲆ + # CYRILLIC SMALL LETTER YAT, CYRILLIC SMALL LETTER TALL YAT + (0x463, 0x1c87), # ѣᲇ + # CYRILLIC SMALL LETTER UNBLENDED UK, CYRILLIC SMALL LETTER MONOGRAPH UK + (0x1c88, 0xa64b), # ᲈꙋ # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE (0x1e61, 0x1e9b), # ṡẛ # LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST @@ -339,11 +355,20 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): charmap += b'\0' * 0xff00 continue # Character set contains non-BMP character codes. + # For range, all BMP characters in the range are already + # proceeded. if fixup: hascased = True - # There are only two ranges of cased non-BMP characters: - # 10400-1044F (Deseret) and 118A0-118DF (Warang Citi), - # and for both ranges RANGE_UNI_IGNORE works. + # For now, IN_IGNORE and RANGE_UNI_IGNORE work for all + # non-BMP characters, because for all non-BMP characters: + # + # 1) c.upper() and c.lower() are non-BMP characters. + # 2) c.lower() == c2.lower() if and only if c equals + # (ignoring case) to c2. + # 3) c.lower().upper() == c.upper(). + # 4) c.lower() in the range R or c.upper() in the range R + # if and only if c equals (ignoring case) to any + # character in the range R. if op is RANGE: op = RANGE_UNI_IGNORE tail.append((op, av)) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 959582e2f12575..dab24bd491d9ab 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -859,16 +859,26 @@ def test_ignore_case(self): self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa") - assert '\u212a'.lower() == 'k' # 'K' + assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'K', '\u212a', re.I)) self.assertTrue(re.match(r'k', '\u212a', re.I)) self.assertTrue(re.match(r'\u212a', 'K', re.I)) self.assertTrue(re.match(r'\u212a', 'k', re.I)) - assert '\u017f'.upper() == 'S' # 'ſ' + + assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ' self.assertTrue(re.match(r'S', '\u017f', re.I)) self.assertTrue(re.match(r's', '\u017f', re.I)) self.assertTrue(re.match(r'\u017f', 'S', re.I)) self.assertTrue(re.match(r'\u017f', 's', re.I)) + + assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В' + self.assertTrue(re.match(r'\u0412', '\u0432', re.I)) + self.assertTrue(re.match(r'\u0412', '\u1c80', re.I)) + self.assertTrue(re.match(r'\u0432', '\u0412', re.I)) + self.assertTrue(re.match(r'\u0432', '\u1c80', re.I)) + self.assertTrue(re.match(r'\u1c80', '\u0412', re.I)) + self.assertTrue(re.match(r'\u1c80', '\u0432', re.I)) + assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I)) self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I)) @@ -882,16 +892,27 @@ def test_ignore_case_set(self): self.assertTrue(re.match(br'[19a]', b'a', re.I)) self.assertTrue(re.match(br'[19a]', b'A', re.I)) self.assertTrue(re.match(br'[19A]', b'a', re.I)) - assert '\u212a'.lower() == 'k' # 'K' + + assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'[19K]', '\u212a', re.I)) self.assertTrue(re.match(r'[19k]', '\u212a', re.I)) self.assertTrue(re.match(r'[19\u212a]', 'K', re.I)) self.assertTrue(re.match(r'[19\u212a]', 'k', re.I)) - assert '\u017f'.upper() == 'S' # 'ſ' + + assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ' self.assertTrue(re.match(r'[19S]', '\u017f', re.I)) self.assertTrue(re.match(r'[19s]', '\u017f', re.I)) self.assertTrue(re.match(r'[19\u017f]', 'S', re.I)) self.assertTrue(re.match(r'[19\u017f]', 's', re.I)) + + assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В' + self.assertTrue(re.match(r'[19\u0412]', '\u0432', re.I)) + self.assertTrue(re.match(r'[19\u0412]', '\u1c80', re.I)) + self.assertTrue(re.match(r'[19\u0432]', '\u0412', re.I)) + self.assertTrue(re.match(r'[19\u0432]', '\u1c80', re.I)) + self.assertTrue(re.match(r'[19\u1c80]', '\u0412', re.I)) + self.assertTrue(re.match(r'[19\u1c80]', '\u0432', re.I)) + assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I)) self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I)) @@ -915,16 +936,26 @@ def test_ignore_case_range(self): self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) - assert '\u212a'.lower() == 'k' # 'K' + assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'[J-M]', '\u212a', re.I)) self.assertTrue(re.match(r'[j-m]', '\u212a', re.I)) self.assertTrue(re.match(r'[\u2129-\u212b]', 'K', re.I)) self.assertTrue(re.match(r'[\u2129-\u212b]', 'k', re.I)) - assert '\u017f'.upper() == 'S' # 'ſ' + + assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ' self.assertTrue(re.match(r'[R-T]', '\u017f', re.I)) self.assertTrue(re.match(r'[r-t]', '\u017f', re.I)) self.assertTrue(re.match(r'[\u017e-\u0180]', 'S', re.I)) self.assertTrue(re.match(r'[\u017e-\u0180]', 's', re.I)) + + assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В' + self.assertTrue(re.match(r'[\u0411-\u0413]', '\u0432', re.I)) + self.assertTrue(re.match(r'[\u0411-\u0413]', '\u1c80', re.I)) + self.assertTrue(re.match(r'[\u0431-\u0433]', '\u0412', re.I)) + self.assertTrue(re.match(r'[\u0431-\u0433]', '\u1c80', re.I)) + self.assertTrue(re.match(r'[\u1c80-\u1c82]', '\u0412', re.I)) + self.assertTrue(re.match(r'[\u1c80-\u1c82]', '\u0432', re.I)) + assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' self.assertTrue(re.match(r'[\ufb04-\ufb05]', '\ufb06', re.I)) self.assertTrue(re.match(r'[\ufb06-\ufb07]', '\ufb05', re.I)) diff --git a/Misc/NEWS.d/next/Library/2022-04-15-18-38-21.gh-issue-91575.fSyAxS.rst b/Misc/NEWS.d/next/Library/2022-04-15-18-38-21.gh-issue-91575.fSyAxS.rst new file mode 100644 index 00000000000000..ba046f2b4d61cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-15-18-38-21.gh-issue-91575.fSyAxS.rst @@ -0,0 +1,2 @@ +Update case-insensitive matching in the :mod:`re` module to the latest +Unicode version. From 92e608d9ad837a531303f015e17e30c0894ac376 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Apr 2022 14:19:27 +0300 Subject: [PATCH 2/4] Update the comment. --- Lib/re/_compiler.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 715e03a647c71d..2e82584b34a862 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -359,16 +359,14 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # proceeded. if fixup: hascased = True - # For now, IN_IGNORE and RANGE_UNI_IGNORE work for all - # non-BMP characters, because for all non-BMP characters: - # - # 1) c.upper() and c.lower() are non-BMP characters. - # 2) c.lower() == c2.lower() if and only if c equals - # (ignoring case) to c2. - # 3) c.lower().upper() == c.upper(). - # 4) c.lower() in the range R or c.upper() in the range R - # if and only if c equals (ignoring case) to any - # character in the range R. + # For now, IN_IGNORE+LITERAL and IN_IGNORE+RANGE_UNI_IGNORE + # work for all non-BMP characters, because two characters + # (at least one of which is not in the BMP) match + # case-insensitively if and only if: + # 1) c1.lower() == c2.lower() + # 2) c1.lower() == c2 or c1.lower().upper() == c2 + # Also, both c.lower() and c.lower().upper() are single + # characters for every non-BMP character. if op is RANGE: op = RANGE_UNI_IGNORE tail.append((op, av)) From d8901e065f6630c852362a377d0e245301f840dd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 17 Apr 2022 16:47:01 +0300 Subject: [PATCH 3/4] Add more comments. --- Lib/test/test_re.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index dab24bd491d9ab..a1c27c9bc0f364 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -859,18 +859,21 @@ def test_ignore_case(self): self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa") + # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'K', '\u212a', re.I)) self.assertTrue(re.match(r'k', '\u212a', re.I)) self.assertTrue(re.match(r'\u212a', 'K', re.I)) self.assertTrue(re.match(r'\u212a', 'k', re.I)) + # Two different characters have the same uppercase. assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ' self.assertTrue(re.match(r'S', '\u017f', re.I)) self.assertTrue(re.match(r's', '\u017f', re.I)) self.assertTrue(re.match(r'\u017f', 'S', re.I)) self.assertTrue(re.match(r'\u017f', 's', re.I)) + # Two different characters have the same uppercase. Unicode 9.0+. assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В' self.assertTrue(re.match(r'\u0412', '\u0432', re.I)) self.assertTrue(re.match(r'\u0412', '\u1c80', re.I)) @@ -879,6 +882,7 @@ def test_ignore_case(self): self.assertTrue(re.match(r'\u1c80', '\u0412', re.I)) self.assertTrue(re.match(r'\u1c80', '\u0432', re.I)) + # Two different characters have the same multicharacter uppercase. assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I)) self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I)) @@ -893,18 +897,21 @@ def test_ignore_case_set(self): self.assertTrue(re.match(br'[19a]', b'A', re.I)) self.assertTrue(re.match(br'[19A]', b'a', re.I)) + # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'[19K]', '\u212a', re.I)) self.assertTrue(re.match(r'[19k]', '\u212a', re.I)) self.assertTrue(re.match(r'[19\u212a]', 'K', re.I)) self.assertTrue(re.match(r'[19\u212a]', 'k', re.I)) + # Two different characters have the same uppercase. assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ' self.assertTrue(re.match(r'[19S]', '\u017f', re.I)) self.assertTrue(re.match(r'[19s]', '\u017f', re.I)) self.assertTrue(re.match(r'[19\u017f]', 'S', re.I)) self.assertTrue(re.match(r'[19\u017f]', 's', re.I)) + # Two different characters have the same uppercase. Unicode 9.0+. assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В' self.assertTrue(re.match(r'[19\u0412]', '\u0432', re.I)) self.assertTrue(re.match(r'[19\u0412]', '\u1c80', re.I)) @@ -913,6 +920,7 @@ def test_ignore_case_set(self): self.assertTrue(re.match(r'[19\u1c80]', '\u0412', re.I)) self.assertTrue(re.match(r'[19\u1c80]', '\u0432', re.I)) + # Two different characters have the same multicharacter uppercase. assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I)) self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I)) @@ -936,18 +944,21 @@ def test_ignore_case_range(self): self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) + # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'[J-M]', '\u212a', re.I)) self.assertTrue(re.match(r'[j-m]', '\u212a', re.I)) self.assertTrue(re.match(r'[\u2129-\u212b]', 'K', re.I)) self.assertTrue(re.match(r'[\u2129-\u212b]', 'k', re.I)) + # Two different characters have the same uppercase. assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ' self.assertTrue(re.match(r'[R-T]', '\u017f', re.I)) self.assertTrue(re.match(r'[r-t]', '\u017f', re.I)) self.assertTrue(re.match(r'[\u017e-\u0180]', 'S', re.I)) self.assertTrue(re.match(r'[\u017e-\u0180]', 's', re.I)) + # Two different characters have the same uppercase. Unicode 9.0+. assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В' self.assertTrue(re.match(r'[\u0411-\u0413]', '\u0432', re.I)) self.assertTrue(re.match(r'[\u0411-\u0413]', '\u1c80', re.I)) @@ -956,6 +967,7 @@ def test_ignore_case_range(self): self.assertTrue(re.match(r'[\u1c80-\u1c82]', '\u0412', re.I)) self.assertTrue(re.match(r'[\u1c80-\u1c82]', '\u0432', re.I)) + # Two different characters have the same multicharacter uppercase. assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' self.assertTrue(re.match(r'[\ufb04-\ufb05]', '\ufb06', re.I)) self.assertTrue(re.match(r'[\ufb06-\ufb07]', '\ufb05', re.I)) From 648cb8aae71a3bd44312f1a1d9d025a3c04a1ce5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 17 Apr 2022 17:00:13 +0300 Subject: [PATCH 4/4] Fix opcode names in the code. --- Lib/re/_compiler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 2e82584b34a862..065f6fbd73244e 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -359,10 +359,11 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # proceeded. if fixup: hascased = True - # For now, IN_IGNORE+LITERAL and IN_IGNORE+RANGE_UNI_IGNORE - # work for all non-BMP characters, because two characters - # (at least one of which is not in the BMP) match - # case-insensitively if and only if: + # For now, IN_UNI_IGNORE+LITERAL and + # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP + # characters, because two characters (at least one of + # which is not in the BMP) match case-insensitively + # if and only if: # 1) c1.lower() == c2.lower() # 2) c1.lower() == c2 or c1.lower().upper() == c2 # Also, both c.lower() and c.lower().upper() are single