diff --git a/components/segmenter/src/rule_segmenter.rs b/components/segmenter/src/rule_segmenter.rs index 600ab60b2c4..d8caebb5167 100644 --- a/components/segmenter/src/rule_segmenter.rs +++ b/components/segmenter/src/rule_segmenter.rs @@ -151,6 +151,7 @@ impl<'l, 's, Y: RuleBreakType<'l, 's> + ?Sized> Iterator for RuleBreakIterator<' .get_break_state_from_table(break_state as u8, self.data.eot_property) == NOT_MATCH_RULE { + self.boundary_property = previous_left_prop; self.iter = previous_iter; self.current_pos_data = previous_pos_data; return self.get_current_position(); diff --git a/components/segmenter/tests/word_rule_status.rs b/components/segmenter/tests/word_rule_status.rs index b4d671b03fc..4ae64552b70 100644 --- a/components/segmenter/tests/word_rule_status.rs +++ b/components/segmenter/tests/word_rule_status.rs @@ -36,6 +36,44 @@ fn rule_status() { assert!(iter.is_word_like(), "Number is true"); } +#[test] +fn rule_status_letter_eof() { + let segmenter = + WordSegmenter::try_new_auto_unstable(&icu_testdata::unstable()).expect("Data exists"); + let mut iter = segmenter.segment_str("one."); + + assert_eq!(iter.next(), Some(0), "SOT"); + assert_eq!(iter.word_type(), WordType::None, "none"); + assert!(!iter.is_word_like(), "SOT is false"); + + assert_eq!(iter.next(), Some(3), "after one"); + assert_eq!(iter.word_type(), WordType::Letter, "letter"); + assert!(iter.is_word_like(), "Letter is true"); + + assert_eq!(iter.next(), Some(4), "after full stop"); + assert_eq!(iter.word_type(), WordType::None, "none"); + assert!(!iter.is_word_like(), "None is false"); +} + +#[test] +fn rule_status_numeric_eof() { + let segmenter = + WordSegmenter::try_new_auto_unstable(&icu_testdata::unstable()).expect("Data exists"); + let mut iter = segmenter.segment_str("42."); + + assert_eq!(iter.next(), Some(0), "SOT"); + assert_eq!(iter.word_type(), WordType::None, "none"); + assert!(!iter.is_word_like(), "SOT is false"); + + assert_eq!(iter.next(), Some(2), "after 42"); + assert_eq!(iter.word_type(), WordType::Number, "Number"); + assert!(iter.is_word_like(), "Number is true"); + + assert_eq!(iter.next(), Some(3), "after full stop"); + assert_eq!(iter.word_type(), WordType::None, "none"); + assert!(!iter.is_word_like(), "None is false"); +} + #[test] fn rule_status_th() { let segmenter =