NU IS AL should have no break opportunity (#3548)

* Fix ISxAL case. * Run cargo make bakeddata * Add comment to link this PR in test file.
unicode-org · Jun 22, 2023 · 004581f · 004581f
1 parent d22475d
commit 004581f
Show file tree

Hide file tree

Showing 9 changed files with 30 additions and 20 deletions.
diff --git a/components/segmenter/data/data/macros/segmenter_line_v1.data.rs b/components/segmenter/data/data/macros/segmenter_line_v1.data.rs
diff --git a/components/segmenter/tests/spec_test.rs b/components/segmenter/tests/spec_test.rs
@@ -103,9 +103,8 @@ impl Iterator for TestContentIterator {
     }
 }
 
-#[test]
-fn run_line_break_test() {
-    let test_iter = TestContentIterator::new("./tests/testdata/LineBreakTest.txt");
+fn line_break_test(filename: &str) {
+    let test_iter = TestContentIterator::new(filename);
     let segmenter =
         LineSegmenter::try_new_dictionary_unstable(&icu_testdata::unstable()).expect("Data exists");
     for mut test in test_iter {
@@ -140,6 +139,16 @@ fn run_line_break_test() {
     }
 }
 
+#[test]
+fn run_line_break_test() {
+    line_break_test("./tests/testdata/LineBreakTest.txt");
+}
+
+#[test]
+fn run_line_break_extra_test() {
+    line_break_test("./tests/testdata/LineBreakExtraTest.txt");
+}
+
 #[test]
 fn run_word_break_test() {
     let test_iter = TestContentIterator::new("./tests/testdata/WordBreakTest.txt");

diff --git a/components/segmenter/tests/testdata/LineBreakExtraTest.txt b/components/segmenter/tests/testdata/LineBreakExtraTest.txt
@@ -0,0 +1,7 @@
+# Additional line breaking tests, not in LineBreakTest.txt
+#
+# https://github.com/unicode-org/icu4x/pull/3548
+× 0031 × 003A × 0041 ÷ # × [NU] × [IS] × [AL] ÷
+× 0031 × 003A × 05D0 ÷ # × [NU] × [IS] × [HL] ÷
+× 0031 × 003A ÷ FFFC ÷ # × [NU] × [IS] ÷ [CB] ÷
+× 0031 × 003A ÷ 1F3CA ÷ # × [NU] × [IS] ÷ [EB] ÷
diff --git a/provider/datagen/data/segmenter/rules/line.toml b/provider/datagen/data/segmenter/rules/line.toml
@@ -1139,12 +1139,6 @@ left = [ "LB25_NU_CL" ]
 right = [ "Any" ]
 break_state = true
 
-[[rules]]
-# LB25
-left = [ "LB25_NU_IS" ]
-right = [ "Any" ]
-break_state = true
-
 [[rules]]
 # LB25
 left = [ "LB25_NU_SY" ]
@@ -1192,7 +1186,7 @@ break_state = false
 [[rules]]
 # LB29
 # (LB1 AL = AI, AL, SA or XX)
-left = [ "IS" ]
+left = [ "IS", "LB25_NU_IS" ]
 right = [ "AL", "HL", "AI", "SA", "XX" ]
 break_state = false
 
@@ -1231,7 +1225,7 @@ break_state = false
 
 [[rules]]
 # (LB25)
-left = [ "LB25_NU_CP" ]
+left = [ "LB25_NU_CP", "LB25_NU_IS" ]
 right = [ "Any" ]
 break_state = true
 

diff --git a/provider/repodata/data/json/fingerprints.csv b/provider/repodata/data/json/fingerprints.csv
diff --git a/provider/repodata/data/json/segmenter/line@1/und.json b/provider/repodata/data/json/segmenter/line@1/und.json
diff --git a/provider/testdata/data/baked/macros/segmenter_line_v1.data.rs b/provider/testdata/data/baked/macros/segmenter_line_v1.data.rs
diff --git a/provider/testdata/data/postcard/fingerprints.csv b/provider/testdata/data/postcard/fingerprints.csv
@@ -1559,7 +1559,7 @@ relativetime/short/year@1, und, 62B, 45e258268a1e1f9
 segmenter/dictionary/w_auto@1, ja, 2007113B, f7552d2848b39b0d
 segmenter/dictionary/wl_ext@1, th, 224981B, f2d574736bb1a754
 segmenter/grapheme@1, und, 9078B, 521276f1d6a6e7fb
-segmenter/line@1, und, 18811B, a5a3ac8178b6b335
+segmenter/line@1, und, 18811B, 131b23adbe306490
 segmenter/lstm/wl_auto@1, th, 72034B, c46e2e0c098c1fc1
 segmenter/sentence@1, und, 14402B, 6adb54fd1dae7b09
 segmenter/word@1, und, 14641B, d91c662e2d94f17f

diff --git a/provider/testdata/data/testdata.postcard b/provider/testdata/data/testdata.postcard