simplify API and add slider post-processing

Author: pascalkuthe

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "imara-diff"
-version = "0.1.8"
+version = "0.2.0"
 edition = "2021"
 authors = ["pascalkuthe <pascalkuthe@pm.me>"]
 rust-version = "1.71"
@@ -18,6 +18,7 @@ exclude = [
 
 [dependencies]
 hashbrown = { version = "0.15", default-features = false, features = ["default-hasher", "inline-more"] }
+memchr = "2.7.4"
 
 [features]
 default = ["unified_diff"]

src/histogram.rs

@@ -1,10 +1,7 @@
-use std::ops::Range;
-
 use crate::histogram::lcs::find_lcs;
 use crate::histogram::list_pool::{ListHandle, ListPool};
 use crate::intern::Token;
-use crate::util::{strip_common_postfix, strip_common_prefix};
-use crate::{myers, Sink};
+use crate::myers;
 
 mod lcs;
 mod list_pool;
@@ -16,17 +13,15 @@ struct Histogram {
     pool: ListPool,
 }
 
-pub fn diff<S: Sink>(
-    mut before: &[Token],
-    mut after: &[Token],
+pub fn diff(
+    before: &[Token],
+    after: &[Token],
+    removed: &mut [bool],
+    added: &mut [bool],
     num_tokens: u32,
-    mut sink: S,
-) -> S::Out {
+) {
     let mut histogram = Histogram::new(num_tokens);
-    let prefix = strip_common_prefix(&mut before, &mut after);
-    strip_common_postfix(&mut before, &mut after);
-    histogram.run(before, prefix, after, prefix, &mut sink);
-    sink.finish()
+    histogram.run(before, after, removed, added);
 }
 
 impl Histogram {
@@ -58,73 +53,49 @@ impl Histogram {
     fn run(
         &mut self,
         mut before: &[Token],
-        mut before_off: u32,
         mut after: &[Token],
-        mut after_off: u32,
-        sink: &mut impl Sink,
+        mut removed: &mut [bool],
+        mut added: &mut [bool],
     ) {
         loop {
             if before.is_empty() {
-                if !after.is_empty() {
-                    sink.process_change(
-                        before_off..before_off,
-                        after_off..after_off + after.len() as u32,
-                    );
-                }
+                added.fill(true);
                 return;
             } else if after.is_empty() {
-                sink.process_change(
-                    before_off..before_off + before.len() as u32,
-                    after_off..after_off,
-                );
+                removed.fill(true);
                 return;
             }
 
             self.populate(before);
             match find_lcs(before, after, self) {
                 // no lcs was found, that means that file1 and file2 two have nothing in common
                 Some(lcs) if lcs.len == 0 => {
-                    sink.process_change(
-                        before_off..before_off + before.len() as u32,
-                        after_off..after_off + after.len() as u32,
-                    );
+                    added.fill(true);
+                    removed.fill(true);
                     return;
                 }
                 Some(lcs) => {
                     self.run(
                         &before[..lcs.before_start as usize],
-                        before_off,
                         &after[..lcs.after_start as usize],
-                        after_off,
-                        sink,
+                        &mut removed[..lcs.before_start as usize],
+                        &mut added[..lcs.after_start as usize],
                     );
 
                     // this is equivalent to (tail) recursion but implement as a loop for efficeny reasons
                     let before_end = lcs.before_start + lcs.len;
                     before = &before[before_end as usize..];
-                    before_off += before_end;
+                    removed = &mut removed[before_end as usize..];
 
                     let after_end = lcs.after_start + lcs.len;
                     after = &after[after_end as usize..];
-                    after_off += after_end;
+                    added = &mut added[after_end as usize..];
                 }
                 None => {
                     // we are diffing two extremely large repetitive files
                     // this is a worst case for histogram diff with O(N^2) performance
-                    // fallback to myers to maintain linear time complexity
-                    myers::diff(
-                        before,
-                        after,
-                        0, // not used by myers
-                        |mut before: Range<u32>, mut after: Range<u32>| {
-                            before.start += before_off;
-                            before.end += before_off;
-                            after.start += after_off;
-                            after.end += after_off;
-                            sink.process_change(before, after)
-                        },
-                        false,
-                    );
+                    // fallback to myers to maintain linear time complxity
+                    myers::diff(before, after, removed, added, false);
                     return;
                 }
             }

src/histogram/lcs.rs

@@ -88,7 +88,6 @@ impl LcsSearch {
 
             let mut end1 = token_idx1 + 1;
             let mut end2 = after_pos + 1;
-
             loop {
                 let token1 = before.get(end1 as usize);
                 let token2 = after.get(end2 as usize);

src/intern.rs

@@ -36,7 +36,7 @@ pub trait TokenSource {
     fn estimate_tokens(&self) -> u32;
 }
 
-/// Two lists of interned [tokens](crate::intern::Token) that can be compared with the [`diff`](crate::diff) function.
+/// Two lists of interned [tokens](crate::intern::Token) that a [`Diff`](crate::Diff) can be computed from.
 ///
 /// A token represents the smallest possible unit of change during a diff.
 /// For text this is usually a line, a word or a single character.
@@ -74,6 +74,23 @@ impl<T: Eq + Hash> InternedInput<T> {
         res
     }
 
+    /// Create an Interner with an intial capacity calculated by calling
+    /// [`estimate_tokens`](crate::intern::TokenSource::estimate_tokens) methods of `before` and `after`
+    pub fn reserve_for_token_source<S: TokenSource<Token = T> + ?Sized>(
+        &mut self,
+        before: &S,
+        after: &S,
+    ) {
+        self.reserve(before.estimate_tokens(), after.estimate_tokens())
+    }
+
+    pub fn reserve(&mut self, capacity_before: u32, capacity_after: u32) {
+        self.before.reserve(capacity_before as usize);
+        self.after.reserve(capacity_after as usize);
+        self.interner
+            .reserve(capacity_before as usize + capacity_after as usize);
+    }
+
     /// replaces `self.before` with the interned Tokens yielded by `input`
     /// Note that this does not erase any tokens from the interner and might therefore be considered
     /// a memory leak. If this function is called often over a long_running process
@@ -133,6 +150,19 @@ impl<T> Interner<T> {
 }
 
 impl<T: Hash + Eq> Interner<T> {
+    /// Create an Interner with an intial capacity calculated by calling
+    /// [`estimate_tokens`](crate::intern::TokenSource::estimate_tokens) methods of `before` and `after`
+    pub fn reserve_for_token_source<S: TokenSource<Token = T>>(&mut self, before: &S, after: &S) {
+        self.reserve(before.estimate_tokens() as usize + after.estimate_tokens() as usize)
+    }
+
+    pub fn reserve(&mut self, capacity: usize) {
+        self.table.reserve(capacity, |&token| {
+            self.hasher.hash_one(&self.tokens[token.0 as usize])
+        });
+        self.tokens.reserve(capacity);
+    }
+
     /// Intern `token` and return a the interned integer.
     pub fn intern(&mut self, token: T) -> Token {
         let hash = self.hasher.hash_one(&token);