UTF-16 support

benches/basic.rs

@@ -15,7 +15,11 @@ extern crate unicode_bidi;
 
 use test::Bencher;
 
-use unicode_bidi::BidiInfo;
+use unicode_bidi::{BidiInfo, BidiInfoU16};
+
+fn to_utf16(s: &str) -> Vec<u16> {
+    s.encode_utf16().collect()
+}
 
 const LTR_TEXTS: &[&str] = &["abc\ndef\nghi", "abc 123\ndef 456\nghi 789"];
 
@@ -29,6 +33,14 @@ fn bench_bidi_info_new(b: &mut Bencher, texts: &[&str]) {
     }
 }
 
+fn bench_bidi_info_new_u16(b: &mut Bencher, texts: &Vec<Vec<u16>>) {
+    for text in texts {
+        b.iter(|| {
+            BidiInfoU16::new(&text, None);
+        });
+    }
+}
+
 fn bench_reorder_line(b: &mut Bencher, texts: &[&str]) {
     for text in texts {
         let bidi_info = BidiInfo::new(text, None);
@@ -41,6 +53,18 @@ fn bench_reorder_line(b: &mut Bencher, texts: &[&str]) {
     }
 }
 
+fn bench_reorder_line_u16(b: &mut Bencher, texts: &Vec<Vec<u16>>) {
+    for text in texts {
+        let bidi_info = BidiInfoU16::new(text, None);
+        b.iter(|| {
+            for para in &bidi_info.paragraphs {
+                let line = para.range.clone();
+                bidi_info.reorder_line(para, line);
+            }
+        });
+    }
+}
+
 #[bench]
 fn bench_1_bidi_info_new_for_ltr_texts(b: &mut Bencher) {
     bench_bidi_info_new(b, LTR_TEXTS);
@@ -60,3 +84,27 @@ fn bench_3_reorder_line_for_ltr_texts(b: &mut Bencher) {
 fn bench_4_reorder_line_for_bidi_texts(b: &mut Bencher) {
     bench_reorder_line(b, BIDI_TEXTS);
 }
+
+#[bench]
+fn bench_5_bidi_info_new_for_ltr_texts_u16(b: &mut Bencher) {
+    let texts_u16: Vec<_> = LTR_TEXTS.iter().map(|t| to_utf16(t)).collect();
+    bench_bidi_info_new_u16(b, &texts_u16);
+}
+
+#[bench]
+fn bench_6_bidi_info_new_for_bidi_texts_u16(b: &mut Bencher) {
+    let texts_u16: Vec<_> = BIDI_TEXTS.iter().map(|t| to_utf16(t)).collect();
+    bench_bidi_info_new_u16(b, &texts_u16);
+}
+
+#[bench]
+fn bench_7_reorder_line_for_ltr_texts_u16(b: &mut Bencher) {
+    let texts_u16: Vec<_> = LTR_TEXTS.iter().map(|t| to_utf16(t)).collect();
+    bench_reorder_line_u16(b, &texts_u16);
+}
+
+#[bench]
+fn bench_8_reorder_line_for_bidi_texts_u16(b: &mut Bencher) {
+    let texts_u16: Vec<_> = BIDI_TEXTS.iter().map(|t| to_utf16(t)).collect();
+    bench_reorder_line_u16(b, &texts_u16);
+}

benches/udhr.rs

@@ -15,7 +15,11 @@ extern crate unicode_bidi;
 
 use test::Bencher;
 
-use unicode_bidi::BidiInfo;
+use unicode_bidi::{BidiInfo, BidiInfoU16};
+
+fn to_utf16(s: &str) -> Vec<u16> {
+    s.encode_utf16().collect()
+}
 
 const LTR_TEXTS: &[&str] = &[
     include_str!("../data/udhr/ltr/udhr_acu_1.txt"),
@@ -55,6 +59,14 @@ fn bench_bidi_info_new(b: &mut Bencher, texts: &[&str]) {
     }
 }
 
+fn bench_bidi_info_new_u16(b: &mut Bencher, texts: &Vec<Vec<u16>>) {
+    for text in texts {
+        b.iter(|| {
+            BidiInfoU16::new(&text, None);
+        });
+    }
+}
+
 fn bench_reorder_line(b: &mut Bencher, texts: &[&str]) {
     for text in texts {
         let bidi_info = BidiInfo::new(text, None);
@@ -67,6 +79,18 @@ fn bench_reorder_line(b: &mut Bencher, texts: &[&str]) {
     }
 }
 
+fn bench_reorder_line_u16(b: &mut Bencher, texts: &Vec<Vec<u16>>) {
+    for text in texts {
+        let bidi_info = BidiInfoU16::new(text, None);
+        b.iter(|| {
+            for para in &bidi_info.paragraphs {
+                let line = para.range.clone();
+                bidi_info.reorder_line(para, line);
+            }
+        });
+    }
+}
+
 #[bench]
 fn bench_1_bidi_info_new_for_ltr_texts(b: &mut Bencher) {
     bench_bidi_info_new(b, LTR_TEXTS);
@@ -86,3 +110,27 @@ fn bench_3_reorder_line_for_ltr_texts(b: &mut Bencher) {
 fn bench_4_reorder_line_for_bidi_texts(b: &mut Bencher) {
     bench_reorder_line(b, BIDI_TEXTS);
 }
+
+#[bench]
+fn bench_5_bidi_info_new_for_ltr_texts_u16(b: &mut Bencher) {
+    let texts_u16: Vec<_> = LTR_TEXTS.iter().map(|t| to_utf16(t)).collect();
+    bench_bidi_info_new_u16(b, &texts_u16);
+}
+
+#[bench]
+fn bench_6_bidi_info_new_for_bidi_texts_u16(b: &mut Bencher) {
+    let texts_u16: Vec<_> = BIDI_TEXTS.iter().map(|t| to_utf16(t)).collect();
+    bench_bidi_info_new_u16(b, &texts_u16);
+}
+
+#[bench]
+fn bench_7_reorder_line_for_ltr_texts_u16(b: &mut Bencher) {
+    let texts_u16: Vec<_> = LTR_TEXTS.iter().map(|t| to_utf16(t)).collect();
+    bench_reorder_line_u16(b, &texts_u16);
+}
+
+#[bench]
+fn bench_8_reorder_line_for_bidi_texts_u16(b: &mut Bencher) {
+    let texts_u16: Vec<_> = BIDI_TEXTS.iter().map(|t| to_utf16(t)).collect();
+    bench_reorder_line_u16(b, &texts_u16);
+}

src/explicit.rs

@@ -18,19 +18,22 @@ use super::char_data::{
     BidiClass::{self, *},
 };
 use super::level::Level;
+use super::TextSource;
 
 /// Compute explicit embedding levels for one paragraph of text (X1-X8).
 ///
 /// `processing_classes[i]` must contain the `BidiClass` of the char at byte index `i`,
 /// for each char in `text`.
 #[cfg_attr(feature = "flame_it", flamer::flame)]
-pub fn compute(
-    text: &str,
+pub fn compute<'a, T: TextSource<'a> + ?Sized>(
+    text: &'a T,
     para_level: Level,
     original_classes: &[BidiClass],
     levels: &mut [Level],
     processing_classes: &mut [BidiClass],
-) {
+) where
+    <T as TextSource<'a>>::IndexLenIter: Iterator<Item = (usize, usize)>,
+{
     assert_eq!(text.len(), original_classes.len());
 
     // <http://www.unicode.org/reports/tr9/#X1>
@@ -41,7 +44,7 @@ pub fn compute(
     let mut overflow_embedding_count = 0u32;
     let mut valid_isolate_count = 0u32;
 
-    for (i, c) in text.char_indices() {
+    for (i, len) in text.indices_lengths() {
         match original_classes[i] {
             // Rules X2-X5c
             RLE | LRE | RLO | LRO | RLI | LRI | FSI => {
@@ -167,7 +170,7 @@ pub fn compute(
         }
 
         // Handle multi-byte characters.
-        for j in 1..c.len_utf8() {
+        for j in 1..len {
             levels[i + j] = levels[i];
             processing_classes[i + j] = processing_classes[i];
         }

src/implicit.rs

@@ -15,14 +15,14 @@ use core::cmp::max;
 use super::char_data::BidiClass::{self, *};
 use super::level::Level;
 use super::prepare::{not_removed_by_x9, IsolatingRunSequence};
-use super::BidiDataSource;
+use super::{BidiDataSource, TextSource};
 
 /// 3.3.4 Resolving Weak Types
 ///
 /// <http://www.unicode.org/reports/tr9/#Resolving_Weak_Types>
 #[cfg_attr(feature = "flame_it", flamer::flame)]
-pub fn resolve_weak(
-    text: &str,
+pub fn resolve_weak<'a, T: TextSource<'a> + ?Sized>(
+    text: &'a T,
     sequence: &IsolatingRunSequence,
     processing_classes: &mut [BidiClass],
 ) {
@@ -120,9 +120,9 @@ pub fn resolve_weak(
                     // See https://github.com/servo/unicode-bidi/issues/86 for improving this.
                     // We want to make sure we check the correct next character by skipping past the rest
                     // of this one.
-                    if let Some(ch) = text.get(i..).and_then(|s| s.chars().next()) {
+                    if let Some((_, char_len)) = text.char_at(i) {
                         let mut next_class = sequence
-                            .iter_forwards_from(i + ch.len_utf8(), run_index)
+                            .iter_forwards_from(i + char_len, run_index)
                             .map(|j| processing_classes[j])
                             // <https://www.unicode.org/reports/tr9/#Retaining_Explicit_Formatting_Characters>
                             .find(not_removed_by_x9)
@@ -156,7 +156,7 @@ pub fn resolve_weak(
                                 }
                                 *class = ON;
                             }
-                            for idx in sequence.iter_forwards_from(i + ch.len_utf8(), run_index) {
+                            for idx in sequence.iter_forwards_from(i + char_len, run_index) {
                                 let class = &mut processing_classes[idx];
                                 if *class != BN {
                                     break;
@@ -248,14 +248,16 @@ pub fn resolve_weak(
 ///
 /// <http://www.unicode.org/reports/tr9/#Resolving_Neutral_Types>
 #[cfg_attr(feature = "flame_it", flamer::flame)]
-pub fn resolve_neutral<D: BidiDataSource>(
-    text: &str,
+pub fn resolve_neutral<'a, D: BidiDataSource, T: TextSource<'a> + ?Sized>(
+    text: &'a T,
     data_source: &D,
     sequence: &IsolatingRunSequence,
     levels: &[Level],
     original_classes: &[BidiClass],
     processing_classes: &mut [BidiClass],
-) {
+) where
+    <T as TextSource<'a>>::CharIter: Iterator<Item = char>,
+{
     // e = embedding direction
     let e: BidiClass = levels[sequence.runs[0].start].bidi_class();
     let not_e = if e == BidiClass::L {
@@ -288,12 +290,13 @@ pub fn resolve_neutral<D: BidiDataSource>(
         let mut found_not_e = false;
         let mut class_to_set = None;
 
-        let start_len_utf8 = text[pair.start..].chars().next().unwrap().len_utf8();
+        let start_char_len =
+            T::char_len(text.subrange(pair.start..pair.end).chars().next().unwrap());
         // > Inspect the bidirectional types of the characters enclosed within the bracket pair.
         //
         // `pair` is [start, end) so we will end up processing the opening character but not the closing one.
         //
-        for enclosed_i in sequence.iter_forwards_from(pair.start + start_len_utf8, pair.start_run) {
+        for enclosed_i in sequence.iter_forwards_from(pair.start + start_char_len, pair.start_run) {
             if enclosed_i >= pair.end {
                 #[cfg(feature = "std")]
                 debug_assert!(
@@ -362,11 +365,12 @@ pub fn resolve_neutral<D: BidiDataSource>(
         if let Some(class_to_set) = class_to_set {
             // Update all processing classes corresponding to the start and end elements, as requested.
             // We should include all bytes of the character, not the first one.
-            let end_len_utf8 = text[pair.end..].chars().next().unwrap().len_utf8();
-            for class in &mut processing_classes[pair.start..pair.start + start_len_utf8] {
+            let end_char_len =
+                T::char_len(text.subrange(pair.end..text.len()).chars().next().unwrap());
+            for class in &mut processing_classes[pair.start..pair.start + start_char_len] {
                 *class = class_to_set;
             }
-            for class in &mut processing_classes[pair.end..pair.end + end_len_utf8] {
+            for class in &mut processing_classes[pair.end..pair.end + end_char_len] {
                 *class = class_to_set;
             }
             // <https://www.unicode.org/reports/tr9/#Retaining_Explicit_Formatting_Characters>
@@ -382,7 +386,7 @@ pub fn resolve_neutral<D: BidiDataSource>(
 
             // This rule deals with sequences of NSMs, so we can just update them all at once, we don't need to worry
             // about character boundaries. We do need to be careful to skip the full set of bytes for the parentheses characters.
-            let nsm_start = pair.start + start_len_utf8;
+            let nsm_start = pair.start + start_char_len;
             for idx in sequence.iter_forwards_from(nsm_start, pair.start_run) {
                 let class = original_classes[idx];
                 if class == BidiClass::NSM || processing_classes[idx] == BN {
@@ -391,7 +395,7 @@ pub fn resolve_neutral<D: BidiDataSource>(
                     break;
                 }
             }
-            let nsm_end = pair.end + end_len_utf8;
+            let nsm_end = pair.end + end_char_len;
             for idx in sequence.iter_forwards_from(nsm_end, pair.end_run) {
                 let class = original_classes[idx];
                 if class == BidiClass::NSM || processing_classes[idx] == BN {
@@ -477,37 +481,30 @@ struct BracketPair {
 /// text source.
 ///
 /// <https://www.unicode.org/reports/tr9/#BD16>
-fn identify_bracket_pairs<D: BidiDataSource>(
-    text: &str,
+fn identify_bracket_pairs<'a, T: TextSource<'a> + ?Sized, D: BidiDataSource>(
+    text: &'a T,
     data_source: &D,
     run_sequence: &IsolatingRunSequence,
     original_classes: &[BidiClass],
-) -> Vec<BracketPair> {
+) -> Vec<BracketPair>
+where
+    <T as TextSource<'a>>::CharIter: Iterator<Item = char>,
+{
     let mut ret = vec![];
     let mut stack = vec![];
 
     for (run_index, level_run) in run_sequence.runs.iter().enumerate() {
-        let slice = if let Some(slice) = text.get(level_run.clone()) {
-            slice
-        } else {
-            #[cfg(feature = "std")]
-            std::debug_assert!(
-                false,
-                "Found broken indices in level run: found indices {}..{} for string of length {}",
-                level_run.start,
-                level_run.end,
-                text.len()
-            );
-            return ret;
-        };
-
-        for (i, ch) in slice.char_indices() {
-            let actual_index = level_run.start + i;
+        let mut actual_index = level_run.start;
+
+        for ch in text.subrange(level_run.clone()).chars() {
+            let char_len = T::char_len(ch);
+
             // All paren characters are ON.
             // From BidiBrackets.txt:
             // > The Unicode property value stability policy guarantees that characters
             // > which have bpt=o or bpt=c also have bc=ON and Bidi_M=Y
-            if original_classes[level_run.start + i] != BidiClass::ON {
+            if original_classes[actual_index] != BidiClass::ON {
+                actual_index += char_len;
                 continue;
             }
 
@@ -552,6 +549,8 @@ fn identify_bracket_pairs<D: BidiDataSource>(
                     }
                 }
             }
+
+            actual_index += char_len;
         }
     }
     // > Sort the list of pairs of text positions in ascending order based on