Don't break word by MinNumLet with Extend. (#4550)

partial progress on #4417
unicode-org · Mar 6, 2024 · bebc6f9 · bebc6f9
1 parent db1772d
commit bebc6f9
Show file tree

Hide file tree

Showing 7 changed files with 202 additions and 47 deletions.
diff --git a/components/segmenter/tests/spec_test.rs b/components/segmenter/tests/spec_test.rs
@@ -175,9 +175,8 @@ fn run_line_break_extra_test() {
     line_break_test(include_str!("testdata/LineBreakExtraTest.txt"));
 }
 
-#[test]
-fn run_word_break_test() {
-    let test_iter = TestContentIterator::new(include_str!("testdata/WordBreakTest.txt"));
+fn word_break_test(file: &'static str) {
+    let test_iter = TestContentIterator::new(file);
     let segmenter = WordSegmenter::new_dictionary();
     for test in test_iter {
         let s: String = test.utf8_vec.into_iter().collect();
@@ -206,6 +205,16 @@ fn run_word_break_test() {
     }
 }
 
+#[test]
+fn run_word_break_test() {
+    word_break_test(include_str!("testdata/WordBreakTest.txt"));
+}
+
+#[test]
+fn run_word_break_extra_test() {
+    word_break_test(include_str!("testdata/WordBreakExtraTest.txt"));
+}
+
 fn grapheme_break_test(file: &'static str) {
     let test_iter = TestContentIterator::new(file);
     let segmenter = GraphemeClusterSegmenter::new();

diff --git a/components/segmenter/tests/testdata/WordBreakExtraTest.txt b/components/segmenter/tests/testdata/WordBreakExtraTest.txt
@@ -0,0 +1,30 @@
+# Additional word breaking tests, not in WordBreakTest.txt
+#
+# https://github.com/unicode-org/icu4x/issues/4417
+÷ 0041 × 002E × 00AD × 0042 ÷ # ÷ ALetter × MidNumLet × Format × ALetter ÷
+÷ 0041 × 002E × 0308 × 0042 ÷ # ÷ ALetter × MidNumLet × Extend × ALetter ÷
+÷ 0041 × 002E × 200D × 0042 ÷ # ÷ ALetter × MidNumLet × ZWJ × ALetter ÷
+÷ 0041 × 002E × 00AD × 05D0 ÷ # ÷ ALetter × MidNumLet × Format × Hebrew_Letter ÷
+÷ 0041 × 002E × 0308 × 05D0 ÷ # ÷ ALetter × MidNumLet × Extend × Hebrew_Letter ÷
+÷ 0041 × 002E × 200D × 05D0 ÷ # ÷ ALetter × MidNumLet × ZWJ × Hebrew_Letter ÷
+÷ 0041 × 0027 × 00AD × 0042 ÷ # ÷ ALetter × Single_Quote × Format × ALetter ÷
+÷ 0041 × 0027 × 0308 × 0042 ÷ # ÷ ALetter × Single_Quote × Extend × ALetter ÷
+÷ 0041 × 0027 × 200D × 0042 ÷ # ÷ ALetter × Single_Quote × ZWJ × ALetter ÷
+÷ 0041 × 0027 × 00AD × 05D0 ÷ # ÷ ALetter × Single_Quote × Format × Hebrew_Letter ÷
+÷ 0041 × 0027 × 0308 × 05D0 ÷ # ÷ ALetter × Single_Quote × Extend × Hebrew_Letter ÷
+÷ 0041 × 0027 × 200D × 05D0 ÷ # ÷ ALetter × Single_Quote × ZWJ × Hebrew_Letter ÷
+÷ 05D0 × 002E × 00AD × 05D0 ÷ # ÷ Hebrew_Letter × MidNumLet × Format × Hebrew_Letter ÷
+÷ 05D0 × 002E × 0308 × 05D0 ÷ # ÷ Hebrew_Letter × MidNumLet × Extend × Hebrew_Letter ÷
+÷ 05D0 × 002E × 200D × 05D0 ÷ # ÷ Hebrew_Letter × MidNumLet × ZWJ × Hebrew_Letter ÷
+÷ 05D0 × 002E × 00AD × 0041 ÷ # ÷ Hebrew_Letter × MidNumLet × Format × ALetter ÷
+÷ 05D0 × 002E × 0308 × 0041 ÷ # ÷ Hebrew_Letter × MidNumLet × Extend × ALetter ÷
+÷ 05D0 × 002E × 200D × 0041 ÷ # ÷ Hebrew_Letter × MidNumLet × ZWJ × ALetter ÷
+÷ 05D0 × 0027 × 00AD × 0041 ÷ # ÷ Hebrew_Letter × Single_Quote × Format × ALetter ÷
+÷ 05D0 × 0027 × 0308 × 0041 ÷ # ÷ Hebrew_Letter × Single_Quote × Extend × ALetter ÷
+÷ 05D0 × 0027 × 200D × 0041 ÷ # ÷ Hebrew_Letter × Single_Quote × ZWJ × ALetter ÷
+÷ 05D0 × 0027 × 00AD × 05D0 ÷ # ÷ Hebrew_Letter × Single_Quote × Format × Hebrew_Letter ÷
+÷ 05D0 × 0027 × 0308 × 05D0 ÷ # ÷ Hebrew_Letter × Single_Quote × Extend × Hebrew_Letter ÷
+÷ 05D0 × 0027 × 200D × 05D0 ÷ # ÷ Hebrew_Letter × Single_Quote × ZWJ × Hebrew_Letter ÷
+÷ 05D0 × 0022 × 00AD × 05D0 ÷ # ÷ Hebrew_Letter × Double_Quote × Format × Hebrew_Letter ÷
+÷ 05D0 × 0022 × 0308 × 05D0 ÷ # ÷ Hebrew_Letter × Double_Quote × Extend × Hebrew_Letter ÷
+÷ 05D0 × 0022 × 200D × 05D0 ÷ # ÷ Hebrew_Letter × Double_Quote × ZWJ × Hebrew_Letter ÷
diff --git a/provider/baked/segmenter/data/macros/segmenter_word_v1.rs.data b/provider/baked/segmenter/data/macros/segmenter_word_v1.rs.data
diff --git a/provider/datagen/data/segmenter/word.toml b/provider/datagen/data/segmenter/word.toml
diff --git a/provider/datagen/tests/data/baked/macros/segmenter_word_v1.rs.data b/provider/datagen/tests/data/baked/macros/segmenter_word_v1.rs.data