From 9ca41b11ee5880e82c912dbf2e78f51549ab5108 Mon Sep 17 00:00:00 2001 From: frederic wagner Date: Wed, 24 Mar 2021 11:30:29 +0100 Subject: [PATCH 01/24] generic tree parallel iterator this commit adds the `descendants` function which is an equivalent of `split` for tree structured data. --- src/iter/descendants.rs | 207 ++++++++++++++++++++++++++++++++++++++++ src/iter/mod.rs | 2 + 2 files changed, 209 insertions(+) create mode 100644 src/iter/descendants.rs diff --git a/src/iter/descendants.rs b/src/iter/descendants.rs new file mode 100644 index 000000000..c34639b2d --- /dev/null +++ b/src/iter/descendants.rs @@ -0,0 +1,207 @@ +use crate::iter::plumbing::{bridge_unindexed, Folder, UnindexedConsumer, UnindexedProducer}; +use crate::prelude::*; +use std::collections::VecDeque; +use std::iter::once; +use std::marker::PhantomData; + +struct DescendantsProducer<'b, S, B, I> { + to_explore: VecDeque>, // we do a depth first exploration so we need a stack + seen: Vec, // nodes which have already been explored + breed: &'b B, // function generating children + phantom: PhantomData, +} + +impl<'b, S, B, I> UnindexedProducer for DescendantsProducer<'b, S, B, I> +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + type Item = S; + fn split(mut self) -> (Self, Option) { + // explore while front is of size one. + while self + .to_explore + .front() + .map(|f| f.len() == 1) + .unwrap_or(false) + { + let front_node = self.to_explore.pop_front().unwrap().pop().unwrap(); + let next_to_explore: Vec<_> = (self.breed)(&front_node).into_iter().collect(); + if !next_to_explore.is_empty() { + self.to_explore.push_back(next_to_explore); + } + self.seen.push(front_node); + } + // now take half of the front. + let f = self.to_explore.front_mut(); + let right_children = f.and_then(|f| split_vec(f)); + let right = right_children + .map(|c| DescendantsProducer { + to_explore: once(c).collect(), + seen: Vec::new(), + breed: self.breed, + phantom: PhantomData, + }) + .or_else(|| { + // we can still try to divide 'seen' + let right_seen = split_vec(&mut self.seen); + right_seen.map(|s| DescendantsProducer { + to_explore: Default::default(), + seen: s, + breed: self.breed, + phantom: PhantomData, + }) + }); + (self, right) + } + fn fold_with(self, mut folder: F) -> F + where + F: Folder, + { + // start by consuming everything seen + for s in self.seen { + folder = folder.consume(s); + if folder.full() { + return folder; + } + } + // now do all remaining explorations + for mut e in self.to_explore { + while let Some(s) = e.pop() { + e.extend((self.breed)(&s)); + folder = folder.consume(s); + if folder.full() { + return folder; + } + } + } + folder + } +} + +/// ParallelIterator for arbitrary tree-shaped patterns. +/// Returned by the [`descendants()`] function. +/// [`descendants()`]: fn.descendants.html +#[derive(Debug)] +pub struct Descendants { + initial_state: S, + breed: B, + phantom: PhantomData, +} + +impl ParallelIterator for Descendants +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + type Item = S; + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = DescendantsProducer { + to_explore: once(vec![self.initial_state]).collect(), + seen: Vec::new(), + breed: &self.breed, + phantom: PhantomData, + }; + bridge_unindexed(producer, consumer) + } +} + +/// Divide given vector in two equally sized vectors. +/// Return `None` if initial size is <=1 +fn split_vec(v: &mut Vec) -> Option> { + if v.len() <= 1 { + None + } else { + let n = v.len() / 2; + + let mut left_vec = Vec::with_capacity(n); + for _ in 0..n { + left_vec.push(v.pop().unwrap()); + } + Some(left_vec) + } +} + +/// Create a tree like parallel iterator from an initial root state. +/// Thre `breed` function should take a state and iterate on all of its children states. +/// +/// # Example +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::descendants; +/// assert_eq!( +/// descendants(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) +/// .sum::(), +/// 12); +/// ``` +/// +/// # Example +/// +/// Here we loop on the following tree: +/// 4 +/// / \ +/// / \ +/// 2 3 +/// / \ +/// / \ +/// 1 2 +/// +/// The best parallelization is obtained when the tree is balanced +/// but we should also be able to handle harder cases. +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::descendants; +/// struct Node { +/// content: u32, +/// left: Option>, +/// right: Option>, +/// } +/// let root = Node { +/// content: 10, +/// left: Some(Box::new(Node { +/// content: 3, +/// left: None, +/// right: None, +/// })), +/// right: Some(Box::new(Node { +/// content: 14, +/// left: None, +/// right: Some(Box::new(Node { +/// content: 18, +/// left: None, +/// right: None, +/// })), +/// })), +/// }; +/// let v: Vec<&Node> = descendants(&root, |r| { +/// r.left +/// .as_ref() +/// .into_iter() +/// .chain(r.right.as_ref()) +/// .map(|n| &**n) +/// }) +/// .collect(); +/// assert_eq!(v.iter().filter(|n| n.content == 10).count(), 1); +/// assert_eq!(v.iter().filter(|n| n.content == 18).count(), 1); +/// assert_eq!(v.len(), 4); +/// ``` +/// +pub fn descendants(root: S, breed: B) -> Descendants +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + Descendants { + initial_state: root, + breed, + phantom: PhantomData, + } +} diff --git a/src/iter/mod.rs b/src/iter/mod.rs index fb44b87f5..6bbbe0005 100644 --- a/src/iter/mod.rs +++ b/src/iter/mod.rs @@ -108,6 +108,7 @@ mod chunks; mod cloned; mod collect; mod copied; +mod descendants; mod empty; mod enumerate; mod extend; @@ -164,6 +165,7 @@ pub use self::{ chunks::Chunks, cloned::Cloned, copied::Copied, + descendants::{descendants, Descendants}, empty::{empty, Empty}, enumerate::Enumerate, filter::Filter, From 97481b0e8d94323e860d6679da88653bb6f4cf9d Mon Sep 17 00:00:00 2001 From: frederic wagner Date: Wed, 24 Mar 2021 11:59:52 +0100 Subject: [PATCH 02/24] descendants: fix for doc --- src/iter/descendants.rs | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/iter/descendants.rs b/src/iter/descendants.rs index c34639b2d..8087df148 100644 --- a/src/iter/descendants.rs +++ b/src/iter/descendants.rs @@ -129,6 +129,8 @@ fn split_vec(v: &mut Vec) -> Option> { /// Create a tree like parallel iterator from an initial root state. /// Thre `breed` function should take a state and iterate on all of its children states. +/// The best parallelization is obtained when the tree is balanced +/// but we should also be able to handle harder cases. /// /// # Example /// @@ -143,26 +145,26 @@ fn split_vec(v: &mut Vec) -> Option> { /// /// # Example /// -/// Here we loop on the following tree: -/// 4 -/// / \ -/// / \ -/// 2 3 -/// / \ -/// / \ -/// 1 2 -/// -/// The best parallelization is obtained when the tree is balanced -/// but we should also be able to handle harder cases. -/// /// ``` /// use rayon::prelude::*; /// use rayon::iter::descendants; +/// /// struct Node { /// content: u32, /// left: Option>, /// right: Option>, /// } +/// +/// // Here we loop on the following tree: +/// // +/// // 10 +/// // / \ +/// // / \ +/// // 3 14 +/// // \ +/// // \ +/// // 18 +/// /// let root = Node { /// content: 10, /// left: Some(Box::new(Node { @@ -180,17 +182,17 @@ fn split_vec(v: &mut Vec) -> Option> { /// })), /// })), /// }; -/// let v: Vec<&Node> = descendants(&root, |r| { +/// let mut v: Vec = descendants(&root, |r| { /// r.left /// .as_ref() /// .into_iter() /// .chain(r.right.as_ref()) /// .map(|n| &**n) /// }) +/// .map(|node| node.content) /// .collect(); -/// assert_eq!(v.iter().filter(|n| n.content == 10).count(), 1); -/// assert_eq!(v.iter().filter(|n| n.content == 18).count(), 1); -/// assert_eq!(v.len(), 4); +/// v.sort_unstable(); +/// assert_eq!(v, vec![3, 10, 14, 18]); /// ``` /// pub fn descendants(root: S, breed: B) -> Descendants From 05eed2a97b113e5ff3fc3203249d9d7904a5dc25 Mon Sep 17 00:00:00 2001 From: frederic wagner Date: Mon, 29 Mar 2021 16:15:52 +0200 Subject: [PATCH 03/24] walk_tree * renaming descendants to walk_tree * preserving order -> this has a performance cost so i'll try to do some benches --- src/iter/mod.rs | 4 +- src/iter/test.rs | 15 +++++++ src/iter/{descendants.rs => walk_tree.rs} | 50 +++++++++++------------ 3 files changed, 42 insertions(+), 27 deletions(-) rename src/iter/{descendants.rs => walk_tree.rs} (80%) diff --git a/src/iter/mod.rs b/src/iter/mod.rs index 6bbbe0005..95ba44dc0 100644 --- a/src/iter/mod.rs +++ b/src/iter/mod.rs @@ -108,7 +108,6 @@ mod chunks; mod cloned; mod collect; mod copied; -mod descendants; mod empty; mod enumerate; mod extend; @@ -156,6 +155,7 @@ mod try_reduce; mod try_reduce_with; mod unzip; mod update; +mod walk_tree; mod while_some; mod zip; mod zip_eq; @@ -165,7 +165,6 @@ pub use self::{ chunks::Chunks, cloned::Cloned, copied::Copied, - descendants::{descendants, Descendants}, empty::{empty, Empty}, enumerate::Enumerate, filter::Filter, @@ -201,6 +200,7 @@ pub use self::{ take_any_while::TakeAnyWhile, try_fold::{TryFold, TryFoldWith}, update::Update, + walk_tree::{walk_tree, WalkTree}, while_some::WhileSome, zip::Zip, zip_eq::ZipEq, diff --git a/src/iter/test.rs b/src/iter/test.rs index 3f9445025..062d2c499 100644 --- a/src/iter/test.rs +++ b/src/iter/test.rs @@ -2240,3 +2240,18 @@ fn check_update() { assert_eq!(v, vec![vec![1, 0], vec![3, 2, 1, 0]]); } + +#[test] +fn walk_tree() { + let v: Vec = crate::iter::walk_tree(0u32..100, |r| { + // root is smallest + let mid = (r.start + 1 + r.end) / 2; + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.start) + .collect(); + assert!(v.into_iter().eq(0..100)); +} diff --git a/src/iter/descendants.rs b/src/iter/walk_tree.rs similarity index 80% rename from src/iter/descendants.rs rename to src/iter/walk_tree.rs index 8087df148..0c5e36fe0 100644 --- a/src/iter/descendants.rs +++ b/src/iter/walk_tree.rs @@ -4,14 +4,15 @@ use std::collections::VecDeque; use std::iter::once; use std::marker::PhantomData; -struct DescendantsProducer<'b, S, B, I> { +#[derive(Debug)] +struct WalkTreeProducer<'b, S, B, I> { to_explore: VecDeque>, // we do a depth first exploration so we need a stack seen: Vec, // nodes which have already been explored breed: &'b B, // function generating children phantom: PhantomData, } -impl<'b, S, B, I> UnindexedProducer for DescendantsProducer<'b, S, B, I> +impl<'b, S, B, I> UnindexedProducer for WalkTreeProducer<'b, S, B, I> where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -37,7 +38,7 @@ where let f = self.to_explore.front_mut(); let right_children = f.and_then(|f| split_vec(f)); let right = right_children - .map(|c| DescendantsProducer { + .map(|c| WalkTreeProducer { to_explore: once(c).collect(), seen: Vec::new(), breed: self.breed, @@ -46,7 +47,7 @@ where .or_else(|| { // we can still try to divide 'seen' let right_seen = split_vec(&mut self.seen); - right_seen.map(|s| DescendantsProducer { + right_seen.map(|s| WalkTreeProducer { to_explore: Default::default(), seen: s, breed: self.breed, @@ -69,7 +70,11 @@ where // now do all remaining explorations for mut e in self.to_explore { while let Some(s) = e.pop() { - e.extend((self.breed)(&s)); + // TODO: this is not very nice, + // in order to maintain order i need + // to reverse the order of children. + let children = (self.breed)(&s).into_iter().collect::>(); + e.extend(children.into_iter().rev()); folder = folder.consume(s); if folder.full() { return folder; @@ -81,16 +86,16 @@ where } /// ParallelIterator for arbitrary tree-shaped patterns. -/// Returned by the [`descendants()`] function. -/// [`descendants()`]: fn.descendants.html +/// Returned by the [`walk_tree()`] function. +/// [`walk_tree()`]: fn.walk_tree.html #[derive(Debug)] -pub struct Descendants { +pub struct WalkTree { initial_state: S, breed: B, phantom: PhantomData, } -impl ParallelIterator for Descendants +impl ParallelIterator for WalkTree where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -101,7 +106,7 @@ where where C: UnindexedConsumer, { - let producer = DescendantsProducer { + let producer = WalkTreeProducer { to_explore: once(vec![self.initial_state]).collect(), seen: Vec::new(), breed: &self.breed, @@ -112,18 +117,14 @@ where } /// Divide given vector in two equally sized vectors. -/// Return `None` if initial size is <=1 +/// Return `None` if initial size is <=1. +/// We return the first half and keep the last half in `v`. fn split_vec(v: &mut Vec) -> Option> { if v.len() <= 1 { None } else { let n = v.len() / 2; - - let mut left_vec = Vec::with_capacity(n); - for _ in 0..n { - left_vec.push(v.pop().unwrap()); - } - Some(left_vec) + Some(v.split_off(n)) } } @@ -136,9 +137,9 @@ fn split_vec(v: &mut Vec) -> Option> { /// /// ``` /// use rayon::prelude::*; -/// use rayon::iter::descendants; +/// use rayon::iter::walk_tree; /// assert_eq!( -/// descendants(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) +/// walk_tree(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) /// .sum::(), /// 12); /// ``` @@ -147,7 +148,7 @@ fn split_vec(v: &mut Vec) -> Option> { /// /// ``` /// use rayon::prelude::*; -/// use rayon::iter::descendants; +/// use rayon::iter::walk_tree; /// /// struct Node { /// content: u32, @@ -182,7 +183,7 @@ fn split_vec(v: &mut Vec) -> Option> { /// })), /// })), /// }; -/// let mut v: Vec = descendants(&root, |r| { +/// let mut v: Vec = walk_tree(&root, |r| { /// r.left /// .as_ref() /// .into_iter() @@ -191,17 +192,16 @@ fn split_vec(v: &mut Vec) -> Option> { /// }) /// .map(|node| node.content) /// .collect(); -/// v.sort_unstable(); -/// assert_eq!(v, vec![3, 10, 14, 18]); +/// assert_eq!(v, vec![10, 3, 14, 18]); /// ``` /// -pub fn descendants(root: S, breed: B) -> Descendants +pub fn walk_tree(root: S, breed: B) -> WalkTree where S: Send, B: Fn(&S) -> I + Send + Sync, I: IntoIterator + Send, { - Descendants { + WalkTree { initial_state: root, breed, phantom: PhantomData, From f6bc3eb73aab7b444b4055a7e357f63643b9dc22 Mon Sep 17 00:00:00 2001 From: Frederic Wagner Date: Wed, 31 Mar 2021 09:08:46 +0200 Subject: [PATCH 04/24] walk tree simplification --- src/iter/walk_tree.rs | 47 ++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 0c5e36fe0..1512ec2b7 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -6,9 +6,9 @@ use std::marker::PhantomData; #[derive(Debug)] struct WalkTreeProducer<'b, S, B, I> { - to_explore: VecDeque>, // we do a depth first exploration so we need a stack - seen: Vec, // nodes which have already been explored - breed: &'b B, // function generating children + to_explore: VecDeque, // nodes (and subtrees) we have to process + seen: Vec, // nodes which have already been explored + breed: &'b B, // function generating children phantom: PhantomData, } @@ -20,26 +20,18 @@ where { type Item = S; fn split(mut self) -> (Self, Option) { + debug_assert!(self.to_explore.len() <= 1); // explore while front is of size one. - while self - .to_explore - .front() - .map(|f| f.len() == 1) - .unwrap_or(false) - { - let front_node = self.to_explore.pop_front().unwrap().pop().unwrap(); - let next_to_explore: Vec<_> = (self.breed)(&front_node).into_iter().collect(); - if !next_to_explore.is_empty() { - self.to_explore.push_back(next_to_explore); - } + while self.to_explore.len() == 1 { + let front_node = self.to_explore.pop_front().unwrap(); + self.to_explore = (self.breed)(&front_node).into_iter().collect(); self.seen.push(front_node); } // now take half of the front. - let f = self.to_explore.front_mut(); - let right_children = f.and_then(|f| split_vec(f)); + let right_children = split_vecdeque(&mut self.to_explore); let right = right_children .map(|c| WalkTreeProducer { - to_explore: once(c).collect(), + to_explore: c, seen: Vec::new(), breed: self.breed, phantom: PhantomData, @@ -68,13 +60,14 @@ where } } // now do all remaining explorations - for mut e in self.to_explore { - while let Some(s) = e.pop() { + for e in self.to_explore { + let mut stack = vec![e]; + while let Some(s) = stack.pop() { // TODO: this is not very nice, // in order to maintain order i need // to reverse the order of children. let children = (self.breed)(&s).into_iter().collect::>(); - e.extend(children.into_iter().rev()); + stack.extend(children.into_iter().rev()); folder = folder.consume(s); if folder.full() { return folder; @@ -107,7 +100,7 @@ where C: UnindexedConsumer, { let producer = WalkTreeProducer { - to_explore: once(vec![self.initial_state]).collect(), + to_explore: once(self.initial_state).collect(), seen: Vec::new(), breed: &self.breed, phantom: PhantomData, @@ -116,6 +109,18 @@ where } } +/// Divide given vector in two equally sized vectors. +/// Return `None` if initial size is <=1. +/// We return the first half and keep the last half in `v`. +fn split_vecdeque(v: &mut VecDeque) -> Option> { + if v.len() <= 1 { + None + } else { + let n = v.len() / 2; + Some(v.split_off(n)) + } +} + /// Divide given vector in two equally sized vectors. /// Return `None` if initial size is <=1. /// We return the first half and keep the last half in `v`. From a07519f4b34f978098d79fff19c10c08905c2792 Mon Sep 17 00:00:00 2001 From: Frederic Wagner Date: Wed, 31 Mar 2021 11:25:41 +0200 Subject: [PATCH 05/24] walk_tree: two functions for two orders sequential folding is recursive i'm not sure how to do it sequentially since this would imply a self borrowing struct (we need to store both the state and the iterator borrowing it) --- examples/tree.rs | 56 +++++++++ src/iter/mod.rs | 2 +- src/iter/test.rs | 4 +- src/iter/walk_tree.rs | 274 +++++++++++++++++++++++++++++++++++++----- 4 files changed, 305 insertions(+), 31 deletions(-) create mode 100644 examples/tree.rs diff --git a/examples/tree.rs b/examples/tree.rs new file mode 100644 index 000000000..24d3573f6 --- /dev/null +++ b/examples/tree.rs @@ -0,0 +1,56 @@ +use rayon::prelude::*; + +const SIZE: u64 = 10_000_000; +const VAL: u64 = SIZE * (SIZE - 1) / 2; + +fn prefix_collect() { + let start = std::time::Instant::now(); + let v: Vec<_> = rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { + // root is smallest + let mid = (r.start + 1 + r.end) / 2; + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.start) + .collect(); + println!("prefix collect took {:?}", start.elapsed()); + assert!(v.into_iter().eq(0..SIZE)); +} +fn prefix_sum() { + let start = std::time::Instant::now(); + let s = rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { + // root is smallest + let mid = (r.start + 1 + r.end) / 2; + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.start) + .sum::(); + println!("prefix sum took {:?}", start.elapsed()); + assert_eq!(s, VAL) +} +fn postfix_sum() { + let start = std::time::Instant::now(); + let s = rayon::iter::walk_tree_postfix(0u64..SIZE, |r| { + // root is smallest + let mid = (r.start + 1 + r.end) / 2; + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.start) + .sum::(); + println!("postfix sum took {:?}", start.elapsed()); + assert_eq!(s, VAL) +} + +fn main() { + prefix_collect(); + prefix_sum(); + postfix_sum(); +} diff --git a/src/iter/mod.rs b/src/iter/mod.rs index 95ba44dc0..cd5c8b853 100644 --- a/src/iter/mod.rs +++ b/src/iter/mod.rs @@ -200,7 +200,7 @@ pub use self::{ take_any_while::TakeAnyWhile, try_fold::{TryFold, TryFoldWith}, update::Update, - walk_tree::{walk_tree, WalkTree}, + walk_tree::{walk_tree_postfix, walk_tree_prefix, WalkTreePostfix, WalkTreePrefix}, while_some::WhileSome, zip::Zip, zip_eq::ZipEq, diff --git a/src/iter/test.rs b/src/iter/test.rs index 062d2c499..5d1027c7e 100644 --- a/src/iter/test.rs +++ b/src/iter/test.rs @@ -2242,8 +2242,8 @@ fn check_update() { } #[test] -fn walk_tree() { - let v: Vec = crate::iter::walk_tree(0u32..100, |r| { +fn walk_tree_prefix() { + let v: Vec = crate::iter::walk_tree_prefix(0u32..100, |r| { // root is smallest let mid = (r.start + 1 + r.end) / 2; // small indices to the left, large to the right diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 1512ec2b7..42e4ef022 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -5,14 +5,14 @@ use std::iter::once; use std::marker::PhantomData; #[derive(Debug)] -struct WalkTreeProducer<'b, S, B, I> { +struct WalkTreePrefixProducer<'b, S, B, I> { to_explore: VecDeque, // nodes (and subtrees) we have to process seen: Vec, // nodes which have already been explored breed: &'b B, // function generating children phantom: PhantomData, } -impl<'b, S, B, I> UnindexedProducer for WalkTreeProducer<'b, S, B, I> +impl<'b, S, B, I> UnindexedProducer for WalkTreePrefixProducer<'b, S, B, I> where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -20,7 +20,6 @@ where { type Item = S; fn split(mut self) -> (Self, Option) { - debug_assert!(self.to_explore.len() <= 1); // explore while front is of size one. while self.to_explore.len() == 1 { let front_node = self.to_explore.pop_front().unwrap(); @@ -30,7 +29,7 @@ where // now take half of the front. let right_children = split_vecdeque(&mut self.to_explore); let right = right_children - .map(|c| WalkTreeProducer { + .map(|c| WalkTreePrefixProducer { to_explore: c, seen: Vec::new(), breed: self.breed, @@ -39,7 +38,7 @@ where .or_else(|| { // we can still try to divide 'seen' let right_seen = split_vec(&mut self.seen); - right_seen.map(|s| WalkTreeProducer { + right_seen.map(|s| WalkTreePrefixProducer { to_explore: Default::default(), seen: s, breed: self.breed, @@ -61,34 +60,249 @@ where } // now do all remaining explorations for e in self.to_explore { - let mut stack = vec![e]; - while let Some(s) = stack.pop() { - // TODO: this is not very nice, - // in order to maintain order i need - // to reverse the order of children. - let children = (self.breed)(&s).into_iter().collect::>(); - stack.extend(children.into_iter().rev()); - folder = folder.consume(s); - if folder.full() { - return folder; + folder = consume_rec_prefix(&self.breed, e, folder); + if folder.full() { + return folder; + } + } + folder + } +} + +fn consume_rec_prefix, S, B: Fn(&S) -> I, I: IntoIterator>( + breed: &B, + s: S, + mut folder: F, +) -> F { + let children = (breed)(&s).into_iter().collect::>(); + folder = folder.consume(s); + if folder.full() { + return folder; + } + for child in children { + folder = consume_rec_prefix(breed, child, folder); + if folder.full() { + return folder; + } + } + folder +} + +/// ParallelIterator for arbitrary tree-shaped patterns. +/// Returned by the [`walk_tree_prefix()`] function. +/// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html +#[derive(Debug)] +pub struct WalkTreePrefix { + initial_state: S, + breed: B, + phantom: PhantomData, +} + +impl ParallelIterator for WalkTreePrefix +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + type Item = S; + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = WalkTreePrefixProducer { + to_explore: once(self.initial_state).collect(), + seen: Vec::new(), + breed: &self.breed, + phantom: PhantomData, + }; + bridge_unindexed(producer, consumer) + } +} + +/// Create a tree like parallel iterator from an initial root state. +/// Thre `breed` function should take a state and iterate on all of its children states. +/// The best parallelization is obtained when the tree is balanced +/// but we should also be able to handle harder cases. +/// +/// This guarantees a prefix ordering. For a postfix ordering see the (faster) [`walk_tree_postfix()`] function. +/// +/// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html +/// +/// # Example +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::walk_tree_prefix; +/// assert_eq!( +/// walk_tree_prefix(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) +/// .sum::(), +/// 12); +/// ``` +/// +/// # Example +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::walk_tree_prefix; +/// +/// struct Node { +/// content: u32, +/// left: Option>, +/// right: Option>, +/// } +/// +/// // Here we loop on the following tree: +/// // +/// // 10 +/// // / \ +/// // / \ +/// // 3 14 +/// // \ +/// // \ +/// // 18 +/// +/// let root = Node { +/// content: 10, +/// left: Some(Box::new(Node { +/// content: 3, +/// left: None, +/// right: None, +/// })), +/// right: Some(Box::new(Node { +/// content: 14, +/// left: None, +/// right: Some(Box::new(Node { +/// content: 18, +/// left: None, +/// right: None, +/// })), +/// })), +/// }; +/// let mut v: Vec = walk_tree_prefix(&root, |r| { +/// r.left +/// .as_ref() +/// .into_iter() +/// .chain(r.right.as_ref()) +/// .map(|n| &**n) +/// }) +/// .map(|node| node.content) +/// .collect(); +/// assert_eq!(v, vec![10, 3, 14, 18]); +/// ``` +/// +pub fn walk_tree_prefix(root: S, breed: B) -> WalkTreePrefix +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + WalkTreePrefix { + initial_state: root, + breed, + phantom: PhantomData, + } +} + +// post fix + +#[derive(Debug)] +struct WalkTreePostfixProducer<'b, S, B, I> { + to_explore: VecDeque, // nodes (and subtrees) we have to process + seen: Vec, // nodes which have already been explored + breed: &'b B, // function generating children + phantom: PhantomData, +} + +impl<'b, S, B, I> UnindexedProducer for WalkTreePostfixProducer<'b, S, B, I> +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + type Item = S; + fn split(mut self) -> (Self, Option) { + // explore while front is of size one. + while self.to_explore.len() == 1 { + let front_node = self.to_explore.pop_front().unwrap(); + self.to_explore = (self.breed)(&front_node).into_iter().collect(); + self.seen.push(front_node); + } + // now take half of the front. + let right_children = split_vecdeque(&mut self.to_explore); + let right = right_children + .map(|c| { + let mut right_seen = Vec::new(); + std::mem::swap(&mut self.seen, &mut right_seen); // postfix -> upper nodes are processed last + WalkTreePostfixProducer { + to_explore: c, + seen: right_seen, + breed: self.breed, + phantom: PhantomData, } + }) + .or_else(|| { + // we can still try to divide 'seen' + let right_seen = split_vec(&mut self.seen); + right_seen.map(|mut s| { + std::mem::swap(&mut self.seen, &mut s); + WalkTreePostfixProducer { + to_explore: Default::default(), + seen: s, + breed: self.breed, + phantom: PhantomData, + } + }) + }); + (self, right) + } + fn fold_with(self, mut folder: F) -> F + where + F: Folder, + { + // now do all remaining explorations + for e in self.to_explore { + folder = consume_rec_postfix(&self.breed, e, folder); + if folder.full() { + return folder; + } + } + // end by consuming everything seen + for s in self.seen.into_iter().rev() { + folder = folder.consume(s); + if folder.full() { + return folder; } } folder } } +fn consume_rec_postfix, S, B: Fn(&S) -> I, I: IntoIterator>( + breed: &B, + s: S, + mut folder: F, +) -> F { + let children = (breed)(&s).into_iter(); + for child in children { + folder = consume_rec_postfix(breed, child, folder); + if folder.full() { + return folder; + } + } + folder.consume(s) +} + /// ParallelIterator for arbitrary tree-shaped patterns. -/// Returned by the [`walk_tree()`] function. -/// [`walk_tree()`]: fn.walk_tree.html +/// Returned by the [`walk_tree_postfix()`] function. +/// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html #[derive(Debug)] -pub struct WalkTree { +pub struct WalkTreePostfix { initial_state: S, breed: B, phantom: PhantomData, } -impl ParallelIterator for WalkTree +impl ParallelIterator for WalkTreePostfix where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -99,7 +313,7 @@ where where C: UnindexedConsumer, { - let producer = WalkTreeProducer { + let producer = WalkTreePostfixProducer { to_explore: once(self.initial_state).collect(), seen: Vec::new(), breed: &self.breed, @@ -109,7 +323,7 @@ where } } -/// Divide given vector in two equally sized vectors. +/// Divide given deque in two equally sized deques. /// Return `None` if initial size is <=1. /// We return the first half and keep the last half in `v`. fn split_vecdeque(v: &mut VecDeque) -> Option> { @@ -138,13 +352,17 @@ fn split_vec(v: &mut Vec) -> Option> { /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// +/// This guarantees a postfix ordering. For a prefix ordering see the (slower) [`walk_tree_prefix()`] function. +/// +/// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html +/// /// # Example /// /// ``` /// use rayon::prelude::*; -/// use rayon::iter::walk_tree; +/// use rayon::iter::walk_tree_postfix; /// assert_eq!( -/// walk_tree(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) +/// walk_tree_postfix(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) /// .sum::(), /// 12); /// ``` @@ -153,7 +371,7 @@ fn split_vec(v: &mut Vec) -> Option> { /// /// ``` /// use rayon::prelude::*; -/// use rayon::iter::walk_tree; +/// use rayon::iter::walk_tree_postfix; /// /// struct Node { /// content: u32, @@ -188,7 +406,7 @@ fn split_vec(v: &mut Vec) -> Option> { /// })), /// })), /// }; -/// let mut v: Vec = walk_tree(&root, |r| { +/// let mut v: Vec = walk_tree_postfix(&root, |r| { /// r.left /// .as_ref() /// .into_iter() @@ -197,16 +415,16 @@ fn split_vec(v: &mut Vec) -> Option> { /// }) /// .map(|node| node.content) /// .collect(); -/// assert_eq!(v, vec![10, 3, 14, 18]); +/// assert_eq!(v, vec![3, 18, 14, 10]); /// ``` /// -pub fn walk_tree(root: S, breed: B) -> WalkTree +pub fn walk_tree_postfix(root: S, breed: B) -> WalkTreePostfix where S: Send, B: Fn(&S) -> I + Send + Sync, I: IntoIterator + Send, { - WalkTree { + WalkTreePostfix { initial_state: root, breed, phantom: PhantomData, From e7c748110e71898dba4ae762b90e37ec080345e5 Mon Sep 17 00:00:00 2001 From: Frederic Wagner Date: Wed, 31 Mar 2021 13:17:21 +0200 Subject: [PATCH 06/24] walk_tree: clean + bench postfix collect test + bench, removed deque only thing needed now is tests for n-ary trees --- examples/tree.rs | 18 ++++++++++++++++++ src/iter/test.rs | 15 +++++++++++++++ src/iter/walk_tree.rs | 33 ++++++++++----------------------- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/examples/tree.rs b/examples/tree.rs index 24d3573f6..cb5110cb2 100644 --- a/examples/tree.rs +++ b/examples/tree.rs @@ -18,6 +18,23 @@ fn prefix_collect() { println!("prefix collect took {:?}", start.elapsed()); assert!(v.into_iter().eq(0..SIZE)); } + +fn postfix_collect() { + let start = std::time::Instant::now(); + let v: Vec<_> = rayon::iter::walk_tree_postfix(0u64..SIZE, |r| { + // root is largest + let mid = (r.start + r.end - 1) / 2; + // small indices to the left, large to the right + std::iter::once(r.start..mid) + .chain(std::iter::once(mid..(r.end - 1))) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.end - 1) + .collect(); + println!("postfix collect took {:?}", start.elapsed()); + assert!(v.into_iter().eq(0..SIZE)); +} + fn prefix_sum() { let start = std::time::Instant::now(); let s = rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { @@ -51,6 +68,7 @@ fn postfix_sum() { fn main() { prefix_collect(); + postfix_collect(); prefix_sum(); postfix_sum(); } diff --git a/src/iter/test.rs b/src/iter/test.rs index 5d1027c7e..d93f29443 100644 --- a/src/iter/test.rs +++ b/src/iter/test.rs @@ -2255,3 +2255,18 @@ fn walk_tree_prefix() { .collect(); assert!(v.into_iter().eq(0..100)); } + +#[test] +fn walk_tree_postfix() { + let v: Vec<_> = crate::iter::walk_tree_postfix(0u64..100, |r| { + // root is largest + let mid = (r.start + r.end - 1) / 2; + // small indices to the left, large to the right + std::iter::once(r.start..mid) + .chain(std::iter::once(mid..(r.end - 1))) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.end - 1) + .collect(); + assert!(v.into_iter().eq(0..100)); +} diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 42e4ef022..491c7423d 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -1,14 +1,13 @@ use crate::iter::plumbing::{bridge_unindexed, Folder, UnindexedConsumer, UnindexedProducer}; use crate::prelude::*; -use std::collections::VecDeque; use std::iter::once; use std::marker::PhantomData; #[derive(Debug)] struct WalkTreePrefixProducer<'b, S, B, I> { - to_explore: VecDeque, // nodes (and subtrees) we have to process - seen: Vec, // nodes which have already been explored - breed: &'b B, // function generating children + to_explore: Vec, // nodes (and subtrees) we have to process + seen: Vec, // nodes which have already been explored + breed: &'b B, // function generating children phantom: PhantomData, } @@ -22,12 +21,12 @@ where fn split(mut self) -> (Self, Option) { // explore while front is of size one. while self.to_explore.len() == 1 { - let front_node = self.to_explore.pop_front().unwrap(); + let front_node = self.to_explore.pop().unwrap(); self.to_explore = (self.breed)(&front_node).into_iter().collect(); self.seen.push(front_node); } // now take half of the front. - let right_children = split_vecdeque(&mut self.to_explore); + let right_children = split_vec(&mut self.to_explore); let right = right_children .map(|c| WalkTreePrefixProducer { to_explore: c, @@ -207,9 +206,9 @@ where #[derive(Debug)] struct WalkTreePostfixProducer<'b, S, B, I> { - to_explore: VecDeque, // nodes (and subtrees) we have to process - seen: Vec, // nodes which have already been explored - breed: &'b B, // function generating children + to_explore: Vec, // nodes (and subtrees) we have to process + seen: Vec, // nodes which have already been explored + breed: &'b B, // function generating children phantom: PhantomData, } @@ -223,12 +222,12 @@ where fn split(mut self) -> (Self, Option) { // explore while front is of size one. while self.to_explore.len() == 1 { - let front_node = self.to_explore.pop_front().unwrap(); + let front_node = self.to_explore.pop().unwrap(); self.to_explore = (self.breed)(&front_node).into_iter().collect(); self.seen.push(front_node); } // now take half of the front. - let right_children = split_vecdeque(&mut self.to_explore); + let right_children = split_vec(&mut self.to_explore); let right = right_children .map(|c| { let mut right_seen = Vec::new(); @@ -323,18 +322,6 @@ where } } -/// Divide given deque in two equally sized deques. -/// Return `None` if initial size is <=1. -/// We return the first half and keep the last half in `v`. -fn split_vecdeque(v: &mut VecDeque) -> Option> { - if v.len() <= 1 { - None - } else { - let n = v.len() / 2; - Some(v.split_off(n)) - } -} - /// Divide given vector in two equally sized vectors. /// Return `None` if initial size is <=1. /// We return the first half and keep the last half in `v`. From b56ae426bca55f057aa1f0f288e55b3123cad592 Mon Sep 17 00:00:00 2001 From: Frederic Wagner Date: Wed, 31 Mar 2021 14:13:00 +0200 Subject: [PATCH 07/24] faster prefix walk tree removed vectors allocations + no recursion --- src/iter/walk_tree.rs | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 491c7423d..b045f251d 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -58,33 +58,26 @@ where } } // now do all remaining explorations + let mut stack = Vec::new(); for e in self.to_explore { - folder = consume_rec_prefix(&self.breed, e, folder); - if folder.full() { - return folder; + stack.push(e); + while let Some(e) = stack.pop() { + extend_reversed(&mut stack, (self.breed)(&e)); + folder = folder.consume(e); + if folder.full() { + return folder; + } } } folder } } -fn consume_rec_prefix, S, B: Fn(&S) -> I, I: IntoIterator>( - breed: &B, - s: S, - mut folder: F, -) -> F { - let children = (breed)(&s).into_iter().collect::>(); - folder = folder.consume(s); - if folder.full() { - return folder; - } - for child in children { - folder = consume_rec_prefix(breed, child, folder); - if folder.full() { - return folder; - } - } - folder +fn extend_reversed>(v: &mut Vec, iter: I) { + let old_size = v.len(); + v.extend(iter); + let new_len = v.len(); + (0..(new_len - old_size) / 2).for_each(|i| v.swap(old_size + i, new_len - i - 1)) } /// ParallelIterator for arbitrary tree-shaped patterns. From e3701d54a55e377e5e640fab94ebbb5de70ae06f Mon Sep 17 00:00:00 2001 From: Frederic Wagner Date: Wed, 31 Mar 2021 15:16:12 +0200 Subject: [PATCH 08/24] faster walk_tree_prefix changed order between siblings for that --- examples/tree.rs | 6 +++--- src/iter/test.rs | 6 +++--- src/iter/walk_tree.rs | 46 ++++++++++++++++++++----------------------- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/examples/tree.rs b/examples/tree.rs index cb5110cb2..80c275a7c 100644 --- a/examples/tree.rs +++ b/examples/tree.rs @@ -8,9 +8,9 @@ fn prefix_collect() { let v: Vec<_> = rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { // root is smallest let mid = (r.start + 1 + r.end) / 2; - // small indices to the left, large to the right - std::iter::once((r.start + 1)..mid) - .chain(std::iter::once(mid..r.end)) + // large indices to the left, small to the right + std::iter::once(mid..r.end) + .chain(std::iter::once((r.start + 1)..mid)) .filter(|r| !r.is_empty()) }) .map(|r| r.start) diff --git a/src/iter/test.rs b/src/iter/test.rs index d93f29443..1025fc8b6 100644 --- a/src/iter/test.rs +++ b/src/iter/test.rs @@ -2246,9 +2246,9 @@ fn walk_tree_prefix() { let v: Vec = crate::iter::walk_tree_prefix(0u32..100, |r| { // root is smallest let mid = (r.start + 1 + r.end) / 2; - // small indices to the left, large to the right - std::iter::once((r.start + 1)..mid) - .chain(std::iter::once(mid..r.end)) + // small indices to the right, large to the left + std::iter::once(mid..r.end) + .chain(std::iter::once((r.start + 1)..mid)) .filter(|r| !r.is_empty()) }) .map(|r| r.start) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index b045f251d..0570e8808 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -28,11 +28,14 @@ where // now take half of the front. let right_children = split_vec(&mut self.to_explore); let right = right_children - .map(|c| WalkTreePrefixProducer { - to_explore: c, - seen: Vec::new(), - breed: self.breed, - phantom: PhantomData, + .map(|mut c| { + std::mem::swap(&mut c, &mut self.to_explore); + WalkTreePrefixProducer { + to_explore: c, + seen: Vec::new(), + breed: self.breed, + phantom: PhantomData, + } }) .or_else(|| { // we can still try to divide 'seen' @@ -46,7 +49,7 @@ where }); (self, right) } - fn fold_with(self, mut folder: F) -> F + fn fold_with(mut self, mut folder: F) -> F where F: Folder, { @@ -58,28 +61,17 @@ where } } // now do all remaining explorations - let mut stack = Vec::new(); - for e in self.to_explore { - stack.push(e); - while let Some(e) = stack.pop() { - extend_reversed(&mut stack, (self.breed)(&e)); - folder = folder.consume(e); - if folder.full() { - return folder; - } + while let Some(e) = self.to_explore.pop() { + self.to_explore.extend((self.breed)(&e)); + folder = folder.consume(e); + if folder.full() { + return folder; } } folder } } -fn extend_reversed>(v: &mut Vec, iter: I) { - let old_size = v.len(); - v.extend(iter); - let new_len = v.len(); - (0..(new_len - old_size) / 2).for_each(|i| v.swap(old_size + i, new_len - i - 1)) -} - /// ParallelIterator for arbitrary tree-shaped patterns. /// Returned by the [`walk_tree_prefix()`] function. /// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html @@ -116,7 +108,9 @@ where /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// -/// This guarantees a prefix ordering. For a postfix ordering see the (faster) [`walk_tree_postfix()`] function. +/// This guarantees a prefix ordering. +/// Between siblings, last bred child comes first. +/// For a postfix ordering see the (faster) [`walk_tree_postfix()`] function. /// /// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html /// @@ -179,7 +173,7 @@ where /// }) /// .map(|node| node.content) /// .collect(); -/// assert_eq!(v, vec![10, 3, 14, 18]); +/// assert_eq!(v, vec![10, 14, 18, 3]); /// ``` /// pub fn walk_tree_prefix(root: S, breed: B) -> WalkTreePrefix @@ -332,7 +326,9 @@ fn split_vec(v: &mut Vec) -> Option> { /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// -/// This guarantees a postfix ordering. For a prefix ordering see the (slower) [`walk_tree_prefix()`] function. +/// This guarantees a postfix ordering. +/// Between siblings, first bred child comes first. +/// For a prefix ordering see the (slower) [`walk_tree_prefix()`] function. /// /// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html /// From 4f214bb8da37af59afeed181adb6b89def11eb60 Mon Sep 17 00:00:00 2001 From: Frederic Wagner Date: Wed, 31 Mar 2021 15:29:25 +0200 Subject: [PATCH 09/24] tree walks : documenting orders --- src/iter/walk_tree.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 0570e8808..469ff7f0a 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -110,6 +110,18 @@ where /// /// This guarantees a prefix ordering. /// Between siblings, last bred child comes first. +/// For example a perfect binary tree of 7 nodes will reduced in the following order: +/// +/// ```text +/// 1 +/// / \ +/// / \ +/// 5 2 +/// / \ / \ +/// 7 6 4 3 +/// ``` +/// +/// /// For a postfix ordering see the (faster) [`walk_tree_postfix()`] function. /// /// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html @@ -328,6 +340,18 @@ fn split_vec(v: &mut Vec) -> Option> { /// /// This guarantees a postfix ordering. /// Between siblings, first bred child comes first. +/// +/// For example a perfect binary tree of 7 nodes will reduced in the following order: +/// +/// ```text +/// 7 +/// / \ +/// / \ +/// 3 6 +/// / \ / \ +/// 1 2 4 5 +/// ``` +/// /// For a prefix ordering see the (slower) [`walk_tree_prefix()`] function. /// /// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html From ef06d796621a70ffaf17cec9d97aac319c461d21 Mon Sep 17 00:00:00 2001 From: wagnerf42 Date: Mon, 12 Apr 2021 10:02:38 +0200 Subject: [PATCH 10/24] Update src/iter/walk_tree.rs Co-authored-by: Niko Matsakis --- src/iter/walk_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 469ff7f0a..387d4be5a 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -104,7 +104,7 @@ where } /// Create a tree like parallel iterator from an initial root state. -/// Thre `breed` function should take a state and iterate on all of its children states. +/// The `children_of` function should take a node and return an iterator over its child nodes. /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// From d6fc8ccb4219234a8784475402db12bd3f9ad478 Mon Sep 17 00:00:00 2001 From: wagnerf42 Date: Mon, 12 Apr 2021 10:02:52 +0200 Subject: [PATCH 11/24] Update src/iter/walk_tree.rs Co-authored-by: Niko Matsakis --- src/iter/walk_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 387d4be5a..becd4e7b7 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -103,7 +103,7 @@ where } } -/// Create a tree like parallel iterator from an initial root state. +/// Create a tree-like parallel iterator from an initial root node. /// The `children_of` function should take a node and return an iterator over its child nodes. /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. From 33adf730771fdcf0350436824761a0c7b48dfcd1 Mon Sep 17 00:00:00 2001 From: wagnerf42 Date: Mon, 12 Apr 2021 10:03:03 +0200 Subject: [PATCH 12/24] Update src/iter/walk_tree.rs Co-authored-by: Niko Matsakis --- src/iter/walk_tree.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index becd4e7b7..044be9e13 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -108,7 +108,9 @@ where /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// -/// This guarantees a prefix ordering. +/// # Ordering +/// +/// This function guarantees a prefix ordering. See also [`walk_tree_postfix`], which guarantees a postfix order. If you don't care about ordering, you should use `walk_tree`, which will use whatever is believed to be fastest. /// Between siblings, last bred child comes first. /// For example a perfect binary tree of 7 nodes will reduced in the following order: /// From 1be2b130035013302656d8e398c3dd72a343ad8a Mon Sep 17 00:00:00 2001 From: wagnerf42 Date: Mon, 12 Apr 2021 10:03:24 +0200 Subject: [PATCH 13/24] Update src/iter/walk_tree.rs Co-authored-by: Niko Matsakis --- src/iter/walk_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 044be9e13..1a90de9d4 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -111,7 +111,7 @@ where /// # Ordering /// /// This function guarantees a prefix ordering. See also [`walk_tree_postfix`], which guarantees a postfix order. If you don't care about ordering, you should use `walk_tree`, which will use whatever is believed to be fastest. -/// Between siblings, last bred child comes first. +/// Between siblings, children are reduced in reverse order -- that is, the children that returned last are reduced first. /// For example a perfect binary tree of 7 nodes will reduced in the following order: /// /// ```text From 2db07ef99f9a38097175fa5ac778c0abeffa7d5a Mon Sep 17 00:00:00 2001 From: frederic wagner Date: Mon, 12 Apr 2021 10:34:27 +0200 Subject: [PATCH 14/24] added walk_tree + doc fixes --- src/iter/mod.rs | 4 +- src/iter/walk_tree.rs | 111 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/src/iter/mod.rs b/src/iter/mod.rs index cd5c8b853..5450cabe1 100644 --- a/src/iter/mod.rs +++ b/src/iter/mod.rs @@ -200,7 +200,9 @@ pub use self::{ take_any_while::TakeAnyWhile, try_fold::{TryFold, TryFoldWith}, update::Update, - walk_tree::{walk_tree_postfix, walk_tree_prefix, WalkTreePostfix, WalkTreePrefix}, + walk_tree::{ + walk_tree, walk_tree_postfix, walk_tree_prefix, WalkTree, WalkTreePostfix, WalkTreePrefix, + }, while_some::WhileSome, zip::Zip, zip_eq::ZipEq, diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 1a90de9d4..9f8afe49e 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -74,6 +74,7 @@ where /// ParallelIterator for arbitrary tree-shaped patterns. /// Returned by the [`walk_tree_prefix()`] function. +/// /// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html #[derive(Debug)] pub struct WalkTreePrefix { @@ -103,14 +104,17 @@ where } } -/// Create a tree-like parallel iterator from an initial root node. +/// Create a tree-like prefix parallel iterator from an initial root node. /// The `children_of` function should take a node and return an iterator over its child nodes. /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// /// # Ordering /// -/// This function guarantees a prefix ordering. See also [`walk_tree_postfix`], which guarantees a postfix order. If you don't care about ordering, you should use `walk_tree`, which will use whatever is believed to be fastest. +/// This function guarantees a prefix ordering. See also [`walk_tree_postfix`], +/// which guarantees a postfix order. +/// If you don't care about ordering, you should use [`walk_tree`], +/// which will use whatever is believed to be fastest. /// Between siblings, children are reduced in reverse order -- that is, the children that returned last are reduced first. /// For example a perfect binary tree of 7 nodes will reduced in the following order: /// @@ -127,9 +131,19 @@ where /// For a postfix ordering see the (faster) [`walk_tree_postfix()`] function. /// /// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html +/// [`walk_tree()`]: fn.walk_tree.html /// /// # Example /// +/// ```text +/// 4 +/// / \ +/// / \ +/// 2 3 +/// / \ +/// 1 2 +/// ``` +/// /// ``` /// use rayon::prelude::*; /// use rayon::iter::walk_tree_prefix; @@ -294,6 +308,7 @@ fn consume_rec_postfix, S, B: Fn(&S) -> I, I: IntoIterator { @@ -335,13 +350,18 @@ fn split_vec(v: &mut Vec) -> Option> { } } -/// Create a tree like parallel iterator from an initial root state. -/// Thre `breed` function should take a state and iterate on all of its children states. +/// Create a tree like postfix parallel iterator from an initial root node. +/// Thre `children_of` function should take a node and iterate on all of its child nodes. /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// -/// This guarantees a postfix ordering. -/// Between siblings, first bred child comes first. +/// # Ordering +/// +/// This function guarantees a postfix ordering. See also [`walk_tree_prefix`] which guarantees a +/// prefix order. If you don't care about ordering, you should use [`walk_tree`], which will use +/// whatever is believed to be fastest. +/// +/// Between siblings, children are reduced in order -- that is first children are reduced first. /// /// For example a perfect binary tree of 7 nodes will reduced in the following order: /// @@ -357,9 +377,19 @@ fn split_vec(v: &mut Vec) -> Option> { /// For a prefix ordering see the (slower) [`walk_tree_prefix()`] function. /// /// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html +/// [`walk_tree()`]: fn.walk_tree.html /// /// # Example /// +/// ```text +/// 4 +/// / \ +/// / \ +/// 2 3 +/// / \ +/// 1 2 +/// ``` +/// /// ``` /// use rayon::prelude::*; /// use rayon::iter::walk_tree_postfix; @@ -432,3 +462,72 @@ where phantom: PhantomData, } } + +/// ParallelIterator for arbitrary tree-shaped patterns. +/// Returned by the [`walk_tree()`] function. +/// +/// [`walk_tree()`]: fn.walk_tree_prefix.html +#[derive(Debug)] +pub struct WalkTree(WalkTreePostfix); + +/// Create a tree like parallel iterator from an initial root node. +/// Thre `children_of` function should take a node and iterate on all of its child nodes. +/// The best parallelization is obtained when the tree is balanced +/// but we should also be able to handle harder cases. +/// +/// # Ordering +/// +/// This function does not guarantee any ordering. +/// See also [`walk_tree_prefix`] which guarantees a +/// prefix order and [`walk_tree_postfix`] which guarantees a postfix order. +/// +/// # Example +/// +/// ```text +/// 4 +/// / \ +/// / \ +/// 2 3 +/// / \ +/// 1 2 +/// ``` +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::walk_tree_postfix; +/// assert_eq!( +/// walk_tree_postfix(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) +/// .sum::(), +/// 12); +/// ``` +/// +/// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html +/// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html +pub fn walk_tree(root: S, breed: B) -> WalkTree +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + let walker = WalkTreePostfix { + initial_state: root, + breed, + phantom: PhantomData, + }; + WalkTree(walker) +} + +impl ParallelIterator for WalkTree +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + type Item = S; + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.0.drive_unindexed(consumer) + } +} From f38e169b06bdb3495742f93a059182c08f4d0c29 Mon Sep 17 00:00:00 2001 From: frederic wagner Date: Mon, 12 Apr 2021 11:06:27 +0200 Subject: [PATCH 15/24] benches for tree walks --- examples/tree.rs | 74 --------------------------------------- rayon-demo/src/main.rs | 2 ++ rayon-demo/src/tree.rs | 79 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 74 deletions(-) delete mode 100644 examples/tree.rs create mode 100644 rayon-demo/src/tree.rs diff --git a/examples/tree.rs b/examples/tree.rs deleted file mode 100644 index 80c275a7c..000000000 --- a/examples/tree.rs +++ /dev/null @@ -1,74 +0,0 @@ -use rayon::prelude::*; - -const SIZE: u64 = 10_000_000; -const VAL: u64 = SIZE * (SIZE - 1) / 2; - -fn prefix_collect() { - let start = std::time::Instant::now(); - let v: Vec<_> = rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { - // root is smallest - let mid = (r.start + 1 + r.end) / 2; - // large indices to the left, small to the right - std::iter::once(mid..r.end) - .chain(std::iter::once((r.start + 1)..mid)) - .filter(|r| !r.is_empty()) - }) - .map(|r| r.start) - .collect(); - println!("prefix collect took {:?}", start.elapsed()); - assert!(v.into_iter().eq(0..SIZE)); -} - -fn postfix_collect() { - let start = std::time::Instant::now(); - let v: Vec<_> = rayon::iter::walk_tree_postfix(0u64..SIZE, |r| { - // root is largest - let mid = (r.start + r.end - 1) / 2; - // small indices to the left, large to the right - std::iter::once(r.start..mid) - .chain(std::iter::once(mid..(r.end - 1))) - .filter(|r| !r.is_empty()) - }) - .map(|r| r.end - 1) - .collect(); - println!("postfix collect took {:?}", start.elapsed()); - assert!(v.into_iter().eq(0..SIZE)); -} - -fn prefix_sum() { - let start = std::time::Instant::now(); - let s = rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { - // root is smallest - let mid = (r.start + 1 + r.end) / 2; - // small indices to the left, large to the right - std::iter::once((r.start + 1)..mid) - .chain(std::iter::once(mid..r.end)) - .filter(|r| !r.is_empty()) - }) - .map(|r| r.start) - .sum::(); - println!("prefix sum took {:?}", start.elapsed()); - assert_eq!(s, VAL) -} -fn postfix_sum() { - let start = std::time::Instant::now(); - let s = rayon::iter::walk_tree_postfix(0u64..SIZE, |r| { - // root is smallest - let mid = (r.start + 1 + r.end) / 2; - // small indices to the left, large to the right - std::iter::once((r.start + 1)..mid) - .chain(std::iter::once(mid..r.end)) - .filter(|r| !r.is_empty()) - }) - .map(|r| r.start) - .sum::(); - println!("postfix sum took {:?}", start.elapsed()); - assert_eq!(s, VAL) -} - -fn main() { - prefix_collect(); - postfix_collect(); - prefix_sum(); - postfix_sum(); -} diff --git a/rayon-demo/src/main.rs b/rayon-demo/src/main.rs index 360d80746..bc7f3fac7 100644 --- a/rayon-demo/src/main.rs +++ b/rayon-demo/src/main.rs @@ -32,6 +32,8 @@ mod sort; #[cfg(test)] mod str_split; #[cfg(test)] +mod tree; +#[cfg(test)] mod vec_collect; #[cfg(test)] diff --git a/rayon-demo/src/tree.rs b/rayon-demo/src/tree.rs new file mode 100644 index 000000000..e85f62708 --- /dev/null +++ b/rayon-demo/src/tree.rs @@ -0,0 +1,79 @@ +//! Some benches for tree walks. +use rayon::prelude::*; + +const SIZE: u64 = 100_000; +const VAL: u64 = SIZE * (SIZE - 1) / 2; + +#[bench] +fn tree_prefix_collect(b: &mut ::test::Bencher) { + let mut vec = None; + b.iter(|| { + vec = Some( + rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { + // root is smallest + let mid = (r.start + 1 + r.end) / 2; + // large indices to the left, small to the right + std::iter::once(mid..r.end) + .chain(std::iter::once((r.start + 1)..mid)) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.start) + .collect::>(), + ) + }); + assert!(vec.unwrap().into_iter().eq(0..SIZE)); +} + +#[bench] +fn tree_postfix_collect(b: &mut ::test::Bencher) { + let mut vec = None; + b.iter(|| { + vec = Some( + rayon::iter::walk_tree_postfix(0u64..SIZE, |r| { + // root is largest + let mid = (r.start + r.end - 1) / 2; + // small indices to the left, large to the right + std::iter::once(r.start..mid) + .chain(std::iter::once(mid..(r.end - 1))) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.end - 1) + .collect::>(), + ) + }); + assert!(vec.unwrap().into_iter().eq(0..SIZE)); +} + +#[bench] +fn tree_prefix_sum(b: &mut ::test::Bencher) { + b.iter(|| { + let s = rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { + // root is smallest + let mid = (r.start + 1 + r.end) / 2; + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.start) + .sum::(); + assert_eq!(s, VAL) + }) +} + +#[bench] +fn tree_postfix_sum(b: &mut ::test::Bencher) { + b.iter(|| { + let s = rayon::iter::walk_tree_postfix(0u64..SIZE, |r| { + // root is smallest + let mid = (r.start + 1 + r.end) / 2; + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.start) + .sum::(); + assert_eq!(s, VAL) + }) +} From d616a7c151fa7fcef3316d4ae33a1feda4e2d259 Mon Sep 17 00:00:00 2001 From: Frederic Wagner Date: Wed, 14 Apr 2021 08:48:39 +0200 Subject: [PATCH 16/24] walk_tree: better documentation --- src/iter/walk_tree.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 9f8afe49e..d55b2de34 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -119,12 +119,15 @@ where /// For example a perfect binary tree of 7 nodes will reduced in the following order: /// /// ```text -/// 1 +/// a /// / \ /// / \ -/// 5 2 +/// b c /// / \ / \ -/// 7 6 4 3 +/// d e f g +/// +/// reduced as a,c,g,f,b,e,d +/// /// ``` /// /// @@ -366,12 +369,15 @@ fn split_vec(v: &mut Vec) -> Option> { /// For example a perfect binary tree of 7 nodes will reduced in the following order: /// /// ```text -/// 7 +/// a /// / \ /// / \ -/// 3 6 +/// b c /// / \ / \ -/// 1 2 4 5 +/// d e f g +/// +/// reduced as d,e,b,f,g,c,a +/// /// ``` /// /// For a prefix ordering see the (slower) [`walk_tree_prefix()`] function. @@ -477,7 +483,8 @@ pub struct WalkTree(WalkTreePostfix); /// /// # Ordering /// -/// This function does not guarantee any ordering. +/// This function does not guarantee any ordering but will +/// use whatever algorithm is thought to achieve the fastest traversal. /// See also [`walk_tree_prefix`] which guarantees a /// prefix order and [`walk_tree_postfix`] which guarantees a postfix order. /// From 5f97d07ca14bdd6545871ddeaa72476c57e6ac82 Mon Sep 17 00:00:00 2001 From: frederic wagner Date: Thu, 15 Apr 2021 21:41:51 +0200 Subject: [PATCH 17/24] walktree : prefix double ended iter - prefix order is reversed, we now required double ended iterators - tests and benches updated accordingly - two more tests for flat trees - removed unneeded malloc in task splitting --- rayon-demo/src/tree.rs | 6 +++--- src/iter/test.rs | 20 +++++++++++++++++--- src/iter/walk_tree.rs | 27 ++++++++++++++++----------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/rayon-demo/src/tree.rs b/rayon-demo/src/tree.rs index e85f62708..1c2c2627e 100644 --- a/rayon-demo/src/tree.rs +++ b/rayon-demo/src/tree.rs @@ -12,9 +12,9 @@ fn tree_prefix_collect(b: &mut ::test::Bencher) { rayon::iter::walk_tree_prefix(0u64..SIZE, |r| { // root is smallest let mid = (r.start + 1 + r.end) / 2; - // large indices to the left, small to the right - std::iter::once(mid..r.end) - .chain(std::iter::once((r.start + 1)..mid)) + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) .filter(|r| !r.is_empty()) }) .map(|r| r.start) diff --git a/src/iter/test.rs b/src/iter/test.rs index 1025fc8b6..77aec2f8a 100644 --- a/src/iter/test.rs +++ b/src/iter/test.rs @@ -2246,9 +2246,9 @@ fn walk_tree_prefix() { let v: Vec = crate::iter::walk_tree_prefix(0u32..100, |r| { // root is smallest let mid = (r.start + 1 + r.end) / 2; - // small indices to the right, large to the left - std::iter::once(mid..r.end) - .chain(std::iter::once((r.start + 1)..mid)) + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) .filter(|r| !r.is_empty()) }) .map(|r| r.start) @@ -2270,3 +2270,17 @@ fn walk_tree_postfix() { .collect(); assert!(v.into_iter().eq(0..100)); } + +#[test] +fn walk_flat_tree_prefix() { + let v: Vec<_> = + crate::iter::walk_tree_prefix(0, |&e| if e < 99 { Some(e + 1) } else { None }).collect(); + assert!(v.into_iter().eq(0..100)); +} + +#[test] +fn walk_flat_tree_postfix() { + let v: Vec<_> = + crate::iter::walk_tree_postfix(99, |&e| if e > 0 { Some(e - 1) } else { None }).collect(); + assert!(v.into_iter().eq(0..100)); +} diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index d55b2de34..9bfe77757 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -11,18 +11,20 @@ struct WalkTreePrefixProducer<'b, S, B, I> { phantom: PhantomData, } -impl<'b, S, B, I> UnindexedProducer for WalkTreePrefixProducer<'b, S, B, I> +impl<'b, S, B, I, IT> UnindexedProducer for WalkTreePrefixProducer<'b, S, B, I> where S: Send, B: Fn(&S) -> I + Send + Sync, - I: IntoIterator + Send, + IT: DoubleEndedIterator, + I: IntoIterator + Send, { type Item = S; fn split(mut self) -> (Self, Option) { // explore while front is of size one. while self.to_explore.len() == 1 { let front_node = self.to_explore.pop().unwrap(); - self.to_explore = (self.breed)(&front_node).into_iter().collect(); + self.to_explore + .extend((self.breed)(&front_node).into_iter().rev()); self.seen.push(front_node); } // now take half of the front. @@ -62,7 +64,7 @@ where } // now do all remaining explorations while let Some(e) = self.to_explore.pop() { - self.to_explore.extend((self.breed)(&e)); + self.to_explore.extend((self.breed)(&e).into_iter().rev()); folder = folder.consume(e); if folder.full() { return folder; @@ -83,11 +85,12 @@ pub struct WalkTreePrefix { phantom: PhantomData, } -impl ParallelIterator for WalkTreePrefix +impl ParallelIterator for WalkTreePrefix where S: Send, B: Fn(&S) -> I + Send + Sync, - I: IntoIterator + Send, + IT: DoubleEndedIterator, + I: IntoIterator + Send, { type Item = S; fn drive_unindexed(self, consumer: C) -> C::Result @@ -126,7 +129,7 @@ where /// / \ / \ /// d e f g /// -/// reduced as a,c,g,f,b,e,d +/// reduced as a,b,d,e,c,f,g /// /// ``` /// @@ -204,14 +207,15 @@ where /// }) /// .map(|node| node.content) /// .collect(); -/// assert_eq!(v, vec![10, 14, 18, 3]); +/// assert_eq!(v, vec![10, 3, 14, 18]); /// ``` /// -pub fn walk_tree_prefix(root: S, breed: B) -> WalkTreePrefix +pub fn walk_tree_prefix(root: S, breed: B) -> WalkTreePrefix where S: Send, B: Fn(&S) -> I + Send + Sync, - I: IntoIterator + Send, + IT: DoubleEndedIterator, + I: IntoIterator + Send, { WalkTreePrefix { initial_state: root, @@ -241,7 +245,8 @@ where // explore while front is of size one. while self.to_explore.len() == 1 { let front_node = self.to_explore.pop().unwrap(); - self.to_explore = (self.breed)(&front_node).into_iter().collect(); + self.to_explore + .extend((self.breed)(&front_node).into_iter()); self.seen.push(front_node); } // now take half of the front. From 4903d5311c9c6db34c67e23e356a58696065be87 Mon Sep 17 00:00:00 2001 From: frederic wagner Date: Fri, 16 Apr 2021 13:11:07 +0200 Subject: [PATCH 18/24] walk_tree: more tests tests for graphs with higher degrees --- src/iter/test.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/iter/test.rs b/src/iter/test.rs index 77aec2f8a..abdd68aa7 100644 --- a/src/iter/test.rs +++ b/src/iter/test.rs @@ -2284,3 +2284,35 @@ fn walk_flat_tree_postfix() { crate::iter::walk_tree_postfix(99, |&e| if e > 0 { Some(e - 1) } else { None }).collect(); assert!(v.into_iter().eq(0..100)); } + +#[test] +fn walk_tree_prefix_degree5() { + let depth = 5; + let nodes_number = (1 - 5i32.pow(depth)) / (1 - 5); + let nodes = (0..nodes_number).collect::>(); + let v: Vec = crate::iter::walk_tree_prefix(nodes.as_slice(), |&r| { + r.split_first() + .into_iter() + .filter_map(|(_, r)| if r.is_empty() { None } else { Some(r) }) + .flat_map(|r| r.chunks(r.len() / 5)) + }) + .filter_map(|r| r.first().copied()) + .collect(); + assert_eq!(v, nodes); +} + +#[test] +fn walk_tree_postfix_degree5() { + let depth = 5; + let nodes_number = (1 - 5i32.pow(depth)) / (1 - 5); + let nodes = (0..nodes_number).collect::>(); + let v: Vec = crate::iter::walk_tree_postfix(nodes.as_slice(), |&r| { + r.split_last() + .into_iter() + .filter_map(|(_, r)| if r.is_empty() { None } else { Some(r) }) + .flat_map(|r| r.chunks(r.len() / 5)) + }) + .filter_map(|r| r.last().copied()) + .collect(); + assert_eq!(v, nodes) +} From 8ce2bdf51369d02ed6a9b33f98b070ca76e1f73f Mon Sep 17 00:00:00 2001 From: frederic wagner Date: Thu, 20 May 2021 10:26:32 +0200 Subject: [PATCH 19/24] walk tree: minor fixes - renamed breed to children_of - doc cleanup - using consume_iter --- src/iter/walk_tree.rs | 72 +++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 40 deletions(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 9bfe77757..c5c96c2f0 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -7,7 +7,7 @@ use std::marker::PhantomData; struct WalkTreePrefixProducer<'b, S, B, I> { to_explore: Vec, // nodes (and subtrees) we have to process seen: Vec, // nodes which have already been explored - breed: &'b B, // function generating children + children_of: &'b B, // function generating children phantom: PhantomData, } @@ -24,7 +24,7 @@ where while self.to_explore.len() == 1 { let front_node = self.to_explore.pop().unwrap(); self.to_explore - .extend((self.breed)(&front_node).into_iter().rev()); + .extend((self.children_of)(&front_node).into_iter().rev()); self.seen.push(front_node); } // now take half of the front. @@ -35,7 +35,7 @@ where WalkTreePrefixProducer { to_explore: c, seen: Vec::new(), - breed: self.breed, + children_of: self.children_of, phantom: PhantomData, } }) @@ -45,7 +45,7 @@ where right_seen.map(|s| WalkTreePrefixProducer { to_explore: Default::default(), seen: s, - breed: self.breed, + children_of: self.children_of, phantom: PhantomData, }) }); @@ -56,15 +56,14 @@ where F: Folder, { // start by consuming everything seen - for s in self.seen { - folder = folder.consume(s); - if folder.full() { - return folder; - } + folder = folder.consume_iter(self.seen); + if folder.full() { + return folder; } // now do all remaining explorations while let Some(e) = self.to_explore.pop() { - self.to_explore.extend((self.breed)(&e).into_iter().rev()); + self.to_explore + .extend((self.children_of)(&e).into_iter().rev()); folder = folder.consume(e); if folder.full() { return folder; @@ -81,7 +80,7 @@ where #[derive(Debug)] pub struct WalkTreePrefix { initial_state: S, - breed: B, + children_of: B, phantom: PhantomData, } @@ -100,7 +99,7 @@ where let producer = WalkTreePrefixProducer { to_explore: once(self.initial_state).collect(), seen: Vec::new(), - breed: &self.breed, + children_of: &self.children_of, phantom: PhantomData, }; bridge_unindexed(producer, consumer) @@ -118,7 +117,6 @@ where /// which guarantees a postfix order. /// If you don't care about ordering, you should use [`walk_tree`], /// which will use whatever is believed to be fastest. -/// Between siblings, children are reduced in reverse order -- that is, the children that returned last are reduced first. /// For example a perfect binary tree of 7 nodes will reduced in the following order: /// /// ```text @@ -210,16 +208,16 @@ where /// assert_eq!(v, vec![10, 3, 14, 18]); /// ``` /// -pub fn walk_tree_prefix(root: S, breed: B) -> WalkTreePrefix +pub fn walk_tree_prefix(root: S, children_of: B) -> WalkTreePrefix where S: Send, B: Fn(&S) -> I + Send + Sync, - IT: DoubleEndedIterator, - I: IntoIterator + Send, + I::IntoIter: DoubleEndedIterator, + I: IntoIterator + Send, { WalkTreePrefix { initial_state: root, - breed, + children_of, phantom: PhantomData, } } @@ -230,7 +228,7 @@ where struct WalkTreePostfixProducer<'b, S, B, I> { to_explore: Vec, // nodes (and subtrees) we have to process seen: Vec, // nodes which have already been explored - breed: &'b B, // function generating children + children_of: &'b B, // function generating children phantom: PhantomData, } @@ -246,7 +244,7 @@ where while self.to_explore.len() == 1 { let front_node = self.to_explore.pop().unwrap(); self.to_explore - .extend((self.breed)(&front_node).into_iter()); + .extend((self.children_of)(&front_node).into_iter()); self.seen.push(front_node); } // now take half of the front. @@ -258,7 +256,7 @@ where WalkTreePostfixProducer { to_explore: c, seen: right_seen, - breed: self.breed, + children_of: self.children_of, phantom: PhantomData, } }) @@ -270,7 +268,7 @@ where WalkTreePostfixProducer { to_explore: Default::default(), seen: s, - breed: self.breed, + children_of: self.children_of, phantom: PhantomData, } }) @@ -283,30 +281,24 @@ where { // now do all remaining explorations for e in self.to_explore { - folder = consume_rec_postfix(&self.breed, e, folder); + folder = consume_rec_postfix(&self.children_of, e, folder); if folder.full() { return folder; } } // end by consuming everything seen - for s in self.seen.into_iter().rev() { - folder = folder.consume(s); - if folder.full() { - return folder; - } - } - folder + folder.consume_iter(self.seen.into_iter().rev()) } } fn consume_rec_postfix, S, B: Fn(&S) -> I, I: IntoIterator>( - breed: &B, + children_of: &B, s: S, mut folder: F, ) -> F { - let children = (breed)(&s).into_iter(); + let children = (children_of)(&s).into_iter(); for child in children { - folder = consume_rec_postfix(breed, child, folder); + folder = consume_rec_postfix(children_of, child, folder); if folder.full() { return folder; } @@ -321,7 +313,7 @@ fn consume_rec_postfix, S, B: Fn(&S) -> I, I: IntoIterator { initial_state: S, - breed: B, + children_of: B, phantom: PhantomData, } @@ -339,7 +331,7 @@ where let producer = WalkTreePostfixProducer { to_explore: once(self.initial_state).collect(), seen: Vec::new(), - breed: &self.breed, + children_of: &self.children_of, phantom: PhantomData, }; bridge_unindexed(producer, consumer) @@ -359,7 +351,7 @@ fn split_vec(v: &mut Vec) -> Option> { } /// Create a tree like postfix parallel iterator from an initial root node. -/// Thre `children_of` function should take a node and iterate on all of its child nodes. +/// The `children_of` function should take a node and iterate on all of its child nodes. /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// @@ -461,7 +453,7 @@ fn split_vec(v: &mut Vec) -> Option> { /// assert_eq!(v, vec![3, 18, 14, 10]); /// ``` /// -pub fn walk_tree_postfix(root: S, breed: B) -> WalkTreePostfix +pub fn walk_tree_postfix(root: S, children_of: B) -> WalkTreePostfix where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -469,7 +461,7 @@ where { WalkTreePostfix { initial_state: root, - breed, + children_of, phantom: PhantomData, } } @@ -482,7 +474,7 @@ where pub struct WalkTree(WalkTreePostfix); /// Create a tree like parallel iterator from an initial root node. -/// Thre `children_of` function should take a node and iterate on all of its child nodes. +/// The `children_of` function should take a node and iterate on all of its child nodes. /// The best parallelization is obtained when the tree is balanced /// but we should also be able to handle harder cases. /// @@ -515,7 +507,7 @@ pub struct WalkTree(WalkTreePostfix); /// /// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html /// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html -pub fn walk_tree(root: S, breed: B) -> WalkTree +pub fn walk_tree(root: S, children_of: B) -> WalkTree where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -523,7 +515,7 @@ where { let walker = WalkTreePostfix { initial_state: root, - breed, + children_of, phantom: PhantomData, }; WalkTree(walker) From 3bf4ae7dba45bce427d4f5ac08d6397fd8f94221 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 9 Feb 2024 16:23:28 -0800 Subject: [PATCH 20/24] walk_tree: reformat documentation --- src/iter/walk_tree.rs | 213 +++++++++++++++++++++--------------------- 1 file changed, 105 insertions(+), 108 deletions(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index c5c96c2f0..4fdf1f88c 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -75,8 +75,6 @@ where /// ParallelIterator for arbitrary tree-shaped patterns. /// Returned by the [`walk_tree_prefix()`] function. -/// -/// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html #[derive(Debug)] pub struct WalkTreePrefix { initial_state: S, @@ -127,16 +125,10 @@ where /// / \ / \ /// d e f g /// -/// reduced as a,b,d,e,c,f,g +/// reduced as a,b,d,e,c,f,g /// /// ``` /// -/// -/// For a postfix ordering see the (faster) [`walk_tree_postfix()`] function. -/// -/// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html -/// [`walk_tree()`]: fn.walk_tree.html -/// /// # Example /// /// ```text @@ -149,25 +141,30 @@ where /// ``` /// /// ``` -/// use rayon::prelude::*; /// use rayon::iter::walk_tree_prefix; -/// assert_eq!( -/// walk_tree_prefix(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) -/// .sum::(), -/// 12); +/// use rayon::prelude::*; +/// +/// let par_iter = walk_tree_prefix(4, |&e| { +/// if e <= 2 { +/// Vec::new() +/// } else { +/// vec![e / 2, e / 2 + 1] +/// } +/// }); +/// assert_eq!(par_iter.sum::(), 12); /// ``` /// /// # Example /// -/// ``` -/// use rayon::prelude::*; -/// use rayon::iter::walk_tree_prefix; +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::walk_tree_prefix; /// -/// struct Node { -/// content: u32, -/// left: Option>, -/// right: Option>, -/// } +/// struct Node { +/// content: u32, +/// left: Option>, +/// right: Option>, +/// } /// /// // Here we loop on the following tree: /// // @@ -179,34 +176,35 @@ where /// // \ /// // 18 /// -/// let root = Node { -/// content: 10, -/// left: Some(Box::new(Node { -/// content: 3, -/// left: None, -/// right: None, -/// })), -/// right: Some(Box::new(Node { -/// content: 14, -/// left: None, -/// right: Some(Box::new(Node { -/// content: 18, -/// left: None, -/// right: None, -/// })), -/// })), -/// }; -/// let mut v: Vec = walk_tree_prefix(&root, |r| { -/// r.left -/// .as_ref() -/// .into_iter() -/// .chain(r.right.as_ref()) -/// .map(|n| &**n) -/// }) -/// .map(|node| node.content) -/// .collect(); -/// assert_eq!(v, vec![10, 3, 14, 18]); -/// ``` +/// let root = Node { +/// content: 10, +/// left: Some(Box::new(Node { +/// content: 3, +/// left: None, +/// right: None, +/// })), +/// right: Some(Box::new(Node { +/// content: 14, +/// left: None, +/// right: Some(Box::new(Node { +/// content: 18, +/// left: None, +/// right: None, +/// })), +/// })), +/// }; +/// +/// let mut v: Vec = walk_tree_prefix(&root, |r| { +/// r.left +/// .as_ref() +/// .into_iter() +/// .chain(r.right.as_ref()) +/// .map(|n| &**n) +/// }) +/// .map(|node| node.content) +/// .collect(); +/// assert_eq!(v, vec![10, 3, 14, 18]); +/// ``` /// pub fn walk_tree_prefix(root: S, children_of: B) -> WalkTreePrefix where @@ -308,8 +306,6 @@ fn consume_rec_postfix, S, B: Fn(&S) -> I, I: IntoIterator { initial_state: S, @@ -377,11 +373,6 @@ fn split_vec(v: &mut Vec) -> Option> { /// /// ``` /// -/// For a prefix ordering see the (slower) [`walk_tree_prefix()`] function. -/// -/// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html -/// [`walk_tree()`]: fn.walk_tree.html -/// /// # Example /// /// ```text @@ -394,25 +385,30 @@ fn split_vec(v: &mut Vec) -> Option> { /// ``` /// /// ``` -/// use rayon::prelude::*; /// use rayon::iter::walk_tree_postfix; -/// assert_eq!( -/// walk_tree_postfix(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) -/// .sum::(), -/// 12); +/// use rayon::prelude::*; +/// +/// let par_iter = walk_tree_postfix(4, |&e| { +/// if e <= 2 { +/// Vec::new() +/// } else { +/// vec![e / 2, e / 2 + 1] +/// } +/// }); +/// assert_eq!(par_iter.sum::(), 12); /// ``` /// /// # Example /// -/// ``` -/// use rayon::prelude::*; -/// use rayon::iter::walk_tree_postfix; +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::walk_tree_postfix; /// -/// struct Node { -/// content: u32, -/// left: Option>, -/// right: Option>, -/// } +/// struct Node { +/// content: u32, +/// left: Option>, +/// right: Option>, +/// } /// /// // Here we loop on the following tree: /// // @@ -424,34 +420,35 @@ fn split_vec(v: &mut Vec) -> Option> { /// // \ /// // 18 /// -/// let root = Node { -/// content: 10, -/// left: Some(Box::new(Node { -/// content: 3, -/// left: None, -/// right: None, -/// })), -/// right: Some(Box::new(Node { -/// content: 14, -/// left: None, -/// right: Some(Box::new(Node { -/// content: 18, -/// left: None, -/// right: None, -/// })), -/// })), -/// }; -/// let mut v: Vec = walk_tree_postfix(&root, |r| { -/// r.left -/// .as_ref() -/// .into_iter() -/// .chain(r.right.as_ref()) -/// .map(|n| &**n) -/// }) -/// .map(|node| node.content) -/// .collect(); -/// assert_eq!(v, vec![3, 18, 14, 10]); -/// ``` +/// let root = Node { +/// content: 10, +/// left: Some(Box::new(Node { +/// content: 3, +/// left: None, +/// right: None, +/// })), +/// right: Some(Box::new(Node { +/// content: 14, +/// left: None, +/// right: Some(Box::new(Node { +/// content: 18, +/// left: None, +/// right: None, +/// })), +/// })), +/// }; +/// +/// let mut v: Vec = walk_tree_postfix(&root, |r| { +/// r.left +/// .as_ref() +/// .into_iter() +/// .chain(r.right.as_ref()) +/// .map(|n| &**n) +/// }) +/// .map(|node| node.content) +/// .collect(); +/// assert_eq!(v, vec![3, 18, 14, 10]); +/// ``` /// pub fn walk_tree_postfix(root: S, children_of: B) -> WalkTreePostfix where @@ -468,8 +465,6 @@ where /// ParallelIterator for arbitrary tree-shaped patterns. /// Returned by the [`walk_tree()`] function. -/// -/// [`walk_tree()`]: fn.walk_tree_prefix.html #[derive(Debug)] pub struct WalkTree(WalkTreePostfix); @@ -497,16 +492,18 @@ pub struct WalkTree(WalkTreePostfix); /// ``` /// /// ``` +/// use rayon::iter::walk_tree; /// use rayon::prelude::*; -/// use rayon::iter::walk_tree_postfix; -/// assert_eq!( -/// walk_tree_postfix(4, |&e| if e <= 2 { Vec::new() } else {vec![e/2, e/2+1]}) -/// .sum::(), -/// 12); -/// ``` /// -/// [`walk_tree_prefix()`]: fn.walk_tree_prefix.html -/// [`walk_tree_postfix()`]: fn.walk_tree_postfix.html +/// let par_iter = walk_tree(4, |&e| { +/// if e <= 2 { +/// Vec::new() +/// } else { +/// vec![e / 2, e / 2 + 1] +/// } +/// }); +/// assert_eq!(par_iter.sum::(), 12); +/// ``` pub fn walk_tree(root: S, children_of: B) -> WalkTree where S: Send, From 825661823ab89c947870d7a3fdc67a1cf85a9be3 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 9 Feb 2024 16:26:00 -0800 Subject: [PATCH 21/24] walk_tree: require doubled-ended for the unordered version Otherwise we'll be under-constrained, unable to actually make a choice between the implementations. (Even though we think the faster one doesn't need that right now.) --- src/iter/walk_tree.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 4fdf1f88c..cc4f16dab 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -509,6 +509,7 @@ where S: Send, B: Fn(&S) -> I + Send + Sync, I: IntoIterator + Send, + I::IntoIter: DoubleEndedIterator, { let walker = WalkTreePostfix { initial_state: root, @@ -523,6 +524,7 @@ where S: Send, B: Fn(&S) -> I + Send + Sync, I: IntoIterator + Send, + I::IntoIter: DoubleEndedIterator, { type Item = S; fn drive_unindexed(self, consumer: C) -> C::Result From 5f708cc1988eaa1c6726ec43d4b4046b30084af1 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 9 Feb 2024 16:34:18 -0800 Subject: [PATCH 22/24] walk_tree: drop `I` parameters from the types --- src/iter/walk_tree.rs | 68 ++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index cc4f16dab..882c07fd8 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -1,24 +1,23 @@ use crate::iter::plumbing::{bridge_unindexed, Folder, UnindexedConsumer, UnindexedProducer}; use crate::prelude::*; use std::iter::once; -use std::marker::PhantomData; #[derive(Debug)] -struct WalkTreePrefixProducer<'b, S, B, I> { +struct WalkTreePrefixProducer<'b, S, B> { to_explore: Vec, // nodes (and subtrees) we have to process seen: Vec, // nodes which have already been explored children_of: &'b B, // function generating children - phantom: PhantomData, } -impl<'b, S, B, I, IT> UnindexedProducer for WalkTreePrefixProducer<'b, S, B, I> +impl UnindexedProducer for WalkTreePrefixProducer<'_, S, B> where S: Send, B: Fn(&S) -> I + Send + Sync, - IT: DoubleEndedIterator, - I: IntoIterator + Send, + I: IntoIterator + Send, + I::IntoIter: DoubleEndedIterator, { type Item = S; + fn split(mut self) -> (Self, Option) { // explore while front is of size one. while self.to_explore.len() == 1 { @@ -36,7 +35,6 @@ where to_explore: c, seen: Vec::new(), children_of: self.children_of, - phantom: PhantomData, } }) .or_else(|| { @@ -46,11 +44,11 @@ where to_explore: Default::default(), seen: s, children_of: self.children_of, - phantom: PhantomData, }) }); (self, right) } + fn fold_with(mut self, mut folder: F) -> F where F: Folder, @@ -76,20 +74,20 @@ where /// ParallelIterator for arbitrary tree-shaped patterns. /// Returned by the [`walk_tree_prefix()`] function. #[derive(Debug)] -pub struct WalkTreePrefix { +pub struct WalkTreePrefix { initial_state: S, children_of: B, - phantom: PhantomData, } -impl ParallelIterator for WalkTreePrefix +impl ParallelIterator for WalkTreePrefix where S: Send, B: Fn(&S) -> I + Send + Sync, - IT: DoubleEndedIterator, - I: IntoIterator + Send, + I: IntoIterator + Send, + I::IntoIter: DoubleEndedIterator, { type Item = S; + fn drive_unindexed(self, consumer: C) -> C::Result where C: UnindexedConsumer, @@ -98,7 +96,6 @@ where to_explore: once(self.initial_state).collect(), seen: Vec::new(), children_of: &self.children_of, - phantom: PhantomData, }; bridge_unindexed(producer, consumer) } @@ -206,37 +203,36 @@ where /// assert_eq!(v, vec![10, 3, 14, 18]); /// ``` /// -pub fn walk_tree_prefix(root: S, children_of: B) -> WalkTreePrefix +pub fn walk_tree_prefix(root: S, children_of: B) -> WalkTreePrefix where S: Send, B: Fn(&S) -> I + Send + Sync, - I::IntoIter: DoubleEndedIterator, I: IntoIterator + Send, + I::IntoIter: DoubleEndedIterator, { WalkTreePrefix { initial_state: root, children_of, - phantom: PhantomData, } } // post fix #[derive(Debug)] -struct WalkTreePostfixProducer<'b, S, B, I> { +struct WalkTreePostfixProducer<'b, S, B> { to_explore: Vec, // nodes (and subtrees) we have to process seen: Vec, // nodes which have already been explored children_of: &'b B, // function generating children - phantom: PhantomData, } -impl<'b, S, B, I> UnindexedProducer for WalkTreePostfixProducer<'b, S, B, I> +impl UnindexedProducer for WalkTreePostfixProducer<'_, S, B> where S: Send, B: Fn(&S) -> I + Send + Sync, I: IntoIterator + Send, { type Item = S; + fn split(mut self) -> (Self, Option) { // explore while front is of size one. while self.to_explore.len() == 1 { @@ -255,7 +251,6 @@ where to_explore: c, seen: right_seen, children_of: self.children_of, - phantom: PhantomData, } }) .or_else(|| { @@ -267,12 +262,12 @@ where to_explore: Default::default(), seen: s, children_of: self.children_of, - phantom: PhantomData, } }) }); (self, right) } + fn fold_with(self, mut folder: F) -> F where F: Folder, @@ -289,11 +284,12 @@ where } } -fn consume_rec_postfix, S, B: Fn(&S) -> I, I: IntoIterator>( - children_of: &B, - s: S, - mut folder: F, -) -> F { +fn consume_rec_postfix(children_of: &B, s: S, mut folder: F) -> F +where + F: Folder, + B: Fn(&S) -> I, + I: IntoIterator, +{ let children = (children_of)(&s).into_iter(); for child in children { folder = consume_rec_postfix(children_of, child, folder); @@ -307,19 +303,19 @@ fn consume_rec_postfix, S, B: Fn(&S) -> I, I: IntoIterator { +pub struct WalkTreePostfix { initial_state: S, children_of: B, - phantom: PhantomData, } -impl ParallelIterator for WalkTreePostfix +impl ParallelIterator for WalkTreePostfix where S: Send, B: Fn(&S) -> I + Send + Sync, I: IntoIterator + Send, { type Item = S; + fn drive_unindexed(self, consumer: C) -> C::Result where C: UnindexedConsumer, @@ -328,7 +324,6 @@ where to_explore: once(self.initial_state).collect(), seen: Vec::new(), children_of: &self.children_of, - phantom: PhantomData, }; bridge_unindexed(producer, consumer) } @@ -450,7 +445,7 @@ fn split_vec(v: &mut Vec) -> Option> { /// assert_eq!(v, vec![3, 18, 14, 10]); /// ``` /// -pub fn walk_tree_postfix(root: S, children_of: B) -> WalkTreePostfix +pub fn walk_tree_postfix(root: S, children_of: B) -> WalkTreePostfix where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -459,14 +454,13 @@ where WalkTreePostfix { initial_state: root, children_of, - phantom: PhantomData, } } /// ParallelIterator for arbitrary tree-shaped patterns. /// Returned by the [`walk_tree()`] function. #[derive(Debug)] -pub struct WalkTree(WalkTreePostfix); +pub struct WalkTree(WalkTreePostfix); /// Create a tree like parallel iterator from an initial root node. /// The `children_of` function should take a node and iterate on all of its child nodes. @@ -504,7 +498,7 @@ pub struct WalkTree(WalkTreePostfix); /// }); /// assert_eq!(par_iter.sum::(), 12); /// ``` -pub fn walk_tree(root: S, children_of: B) -> WalkTree +pub fn walk_tree(root: S, children_of: B) -> WalkTree where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -514,12 +508,11 @@ where let walker = WalkTreePostfix { initial_state: root, children_of, - phantom: PhantomData, }; WalkTree(walker) } -impl ParallelIterator for WalkTree +impl ParallelIterator for WalkTree where S: Send, B: Fn(&S) -> I + Send + Sync, @@ -527,6 +520,7 @@ where I::IntoIter: DoubleEndedIterator, { type Item = S; + fn drive_unindexed(self, consumer: C) -> C::Result where C: UnindexedConsumer, From 306748ca9beabcff376422e4b196de0d671709b3 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 9 Feb 2024 16:35:01 -0800 Subject: [PATCH 23/24] walk_tree: don't require `I: Send` --- src/iter/walk_tree.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index 882c07fd8..f743ca8c9 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -13,7 +13,7 @@ impl UnindexedProducer for WalkTreePrefixProducer<'_, S, B> where S: Send, B: Fn(&S) -> I + Send + Sync, - I: IntoIterator + Send, + I: IntoIterator, I::IntoIter: DoubleEndedIterator, { type Item = S; @@ -83,7 +83,7 @@ impl ParallelIterator for WalkTreePrefix where S: Send, B: Fn(&S) -> I + Send + Sync, - I: IntoIterator + Send, + I: IntoIterator, I::IntoIter: DoubleEndedIterator, { type Item = S; @@ -207,7 +207,7 @@ pub fn walk_tree_prefix(root: S, children_of: B) -> WalkTreePrefix I + Send + Sync, - I: IntoIterator + Send, + I: IntoIterator, I::IntoIter: DoubleEndedIterator, { WalkTreePrefix { @@ -229,7 +229,7 @@ impl UnindexedProducer for WalkTreePostfixProducer<'_, S, B> where S: Send, B: Fn(&S) -> I + Send + Sync, - I: IntoIterator + Send, + I: IntoIterator, { type Item = S; @@ -312,7 +312,7 @@ impl ParallelIterator for WalkTreePostfix where S: Send, B: Fn(&S) -> I + Send + Sync, - I: IntoIterator + Send, + I: IntoIterator, { type Item = S; @@ -449,7 +449,7 @@ pub fn walk_tree_postfix(root: S, children_of: B) -> WalkTreePostfix I + Send + Sync, - I: IntoIterator + Send, + I: IntoIterator, { WalkTreePostfix { initial_state: root, @@ -502,7 +502,7 @@ pub fn walk_tree(root: S, children_of: B) -> WalkTree where S: Send, B: Fn(&S) -> I + Send + Sync, - I: IntoIterator + Send, + I: IntoIterator, I::IntoIter: DoubleEndedIterator, { let walker = WalkTreePostfix { From 96b365c875fd858bafeb13510da14fffec6346b2 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 9 Feb 2024 16:38:56 -0800 Subject: [PATCH 24/24] walk_tree: use `mem::take` --- src/iter/walk_tree.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/iter/walk_tree.rs b/src/iter/walk_tree.rs index f743ca8c9..f7d6ca86d 100644 --- a/src/iter/walk_tree.rs +++ b/src/iter/walk_tree.rs @@ -245,8 +245,7 @@ where let right_children = split_vec(&mut self.to_explore); let right = right_children .map(|c| { - let mut right_seen = Vec::new(); - std::mem::swap(&mut self.seen, &mut right_seen); // postfix -> upper nodes are processed last + let right_seen = std::mem::take(&mut self.seen); // postfix -> upper nodes are processed last WalkTreePostfixProducer { to_explore: c, seen: right_seen,