From 36db8dbb45191df31a51fa9cfcf20df162c3cbb8 Mon Sep 17 00:00:00 2001 From: Peter Stefek Date: Thu, 21 Nov 2019 19:15:50 -0800 Subject: [PATCH 1/7] small optimization --- .../multires_stochastic_texture_synthesis.rs | 234 +++++++++--------- 1 file changed, 121 insertions(+), 113 deletions(-) diff --git a/lib/src/multires_stochastic_texture_synthesis.rs b/lib/src/multires_stochastic_texture_synthesis.rs index d95dd70..aff80c8 100644 --- a/lib/src/multires_stochastic_texture_synthesis.rs +++ b/lib/src/multires_stochastic_texture_synthesis.rs @@ -582,7 +582,6 @@ impl Generator { *output = (n2_coord, n_map_id) } - //record the candidate info candidates_vec[candidate_count].coord = (candidate_coord, n_map_id); candidates_vec[candidate_count].id = (n_patch_id, n_map_id); @@ -811,7 +810,6 @@ impl Generator { for _ in 0..n_workers { scope.spawn(|_| { let mut candidates: Vec = Vec::new(); - let mut candidates_patterns: Vec = Vec::new(); let mut my_pattern: ColorPattern = ColorPattern::new(); let mut k_neighs: Vec = Vec::with_capacity(params.nearest_neighbors as usize); @@ -820,12 +818,9 @@ impl Generator { + params.random_sample_locations as usize; candidates.resize(max_candidate_count, CandidateStruct::default()); - candidates_patterns.resize(max_candidate_count, ColorPattern::new()); //alloc storage for our guides (regardless of whether we have them or not) let mut my_guide_pattern: ColorPattern = ColorPattern::new(); - let mut candidates_guide_patterns: Vec = Vec::new(); - candidates_guide_patterns.resize(max_candidate_count, ColorPattern::new()); let out_color_map = &[ImageBuffer::from(self.color_map.as_ref())]; @@ -886,20 +881,7 @@ impl Generator { loop_seed + 1, ); - // 3.1 get patterns for color maps - for (cand_i, cand) in candidates.iter().enumerate() { - k_neighs_to_color_pattern( - &cand.k_neighs, - image::Rgba([0, 0, 0, 255]), - &example_maps, - &mut candidates_patterns[cand_i], - false, - ); - } - - let candidates_patterns = &candidates_patterns[0..candidates.len()]; - - k_neighs_to_color_pattern( + k_neighs_to_precomputed_reference_pattern( &k_neighs_w_map_id, //feed into the function with always 0 index of the sample map image::Rgba([0, 0, 0, 255]), out_color_map, @@ -907,27 +889,17 @@ impl Generator { is_tiling_mode, ); + // TODO Peter do not forget to move guide cost lookup table // 3.2 get pattern for guide map if we have them let (my_cost, guide_cost) = if let Some(ref in_guides) = guides { - // populate guidance patterns for candidates - for (cand_i, cand) in candidates.iter().enumerate() { - k_neighs_to_color_pattern( - &cand.k_neighs, - image::Rgba([0, 0, 0, 255]), - &in_guides.example_guides, - &mut candidates_guide_patterns[cand_i], - false, - ); - - //get example pattern to compare to - k_neighs_to_color_pattern( - &k_neighs_w_map_id, - image::Rgba([0, 0, 0, 255]), - &[in_guides.target_guide.clone()], - &mut my_guide_pattern, - is_tiling_mode, - ); - } + //get example pattern to compare to + k_neighs_to_precomputed_reference_pattern( + &k_neighs_w_map_id, + image::Rgba([0, 0, 0, 255]), + &[in_guides.target_guide.clone()], + &mut my_guide_pattern, + is_tiling_mode, + ); ( &my_inverse_alpha_cost_precomputed, @@ -937,16 +909,14 @@ impl Generator { (&cauchy_precomputed, None) }; - let candidates_guide_patterns = - &candidates_guide_patterns[0..candidates.len()]; - // 4. find best match based on the candidate patterns let (best_match, score) = find_best_match( + image::Rgba([0, 0, 0, 255]), + &example_maps, + &guides, &candidates, &my_pattern, - &candidates_patterns, &my_guide_pattern, - &candidates_guide_patterns, &k_neighs_dist, &my_cost, guide_cost, @@ -1018,7 +988,49 @@ impl Generator { } } -fn k_neighs_to_color_pattern( +#[inline] +fn metric_cauchy(a: u8, b: u8, sig2: f32) -> f32 { + let mut x2 = (f32::from(a) - f32::from(b)) / 255.0; //normalize the colors to be between 0-1 + x2 = x2 * x2; + (1.0 + x2 / sig2).ln() +} + +#[inline] +fn metric_l2(a: u8, b: u8) -> f32 { + let x = (f32::from(a) - f32::from(b)) / 255.0; + x * x +} + +#[inline] +fn get_color_of_neighbor( + outside_color: image::Rgba, + source_maps: &[ImageBuffer<'_>], + n_coord: SignedCoord2D, + n_map: MapId, + neighbor_color: &mut [u8], + is_wrap_mode: bool, + wrap_dim: (i32, i32), +) { + let coord = if is_wrap_mode { + n_coord.wrap(wrap_dim) + } else { + n_coord + }; + + //check if he haven't gone outside the possible bounds + if source_maps[n_map.0 as usize].is_in_bounds(coord) { + neighbor_color.copy_from_slice( + &(source_maps[n_map.0 as usize]) + .get_pixel(coord.x as u32, coord.y as u32) + .0[..4], + ); + } else { + // if we have gone out of bounds, then just fill as outside color + neighbor_color.copy_from_slice(&outside_color.0[..]); + } +} + +fn k_neighs_to_precomputed_reference_pattern( k_neighs: &[(SignedCoord2D, MapId)], outside_color: image::Rgba, source_maps: &[ImageBuffer<'_>], @@ -1034,50 +1046,30 @@ fn k_neighs_to_color_pattern( ); for (n_coord, n_map) in k_neighs { - let coord = if is_wrap_mode { - n_coord.wrap(wrap_dim) - } else { - *n_coord - }; - let end = i + 4; - //check if he haven't gone outside the possible bounds - if source_maps[n_map.0 as usize].is_in_bounds(coord) { - pattern.0[i..end].copy_from_slice( - &(source_maps[n_map.0 as usize]) - .get_pixel(coord.x as u32, coord.y as u32) - .0[..4], - ) - } else { - // if we have gone out of bounds, then just fill as outside color - pattern.0[i..end].copy_from_slice(&outside_color.0[..]); - } + get_color_of_neighbor( + outside_color, + source_maps, + *n_coord, + *n_map, + &mut (pattern.0[i..end]), + is_wrap_mode, + wrap_dim, + ); i = end; } } -#[inline] -fn metric_cauchy(a: u8, b: u8, sig2: f32) -> f32 { - let mut x2 = (f32::from(a) - f32::from(b)) / 255.0; //normalize the colors to be between 0-1 - x2 = x2 * x2; - (1.0 + x2 / sig2).ln() -} - -#[inline] -fn metric_l2(a: u8, b: u8) -> f32 { - let x = (f32::from(a) - f32::from(b)) / 255.0; - x * x -} - #[allow(clippy::too_many_arguments)] fn find_best_match<'a>( + outside_color: image::Rgba, + source_maps: &[ImageBuffer<'_>], + guides: &Option>, candidates: &'a [CandidateStruct], - my_pattern: &ColorPattern, - candidates_patterns: &[ColorPattern], - my_guide_pattern: &ColorPattern, - candidates_guide_patterns: &[ColorPattern], + my_precomputed_pattern: &ColorPattern, + my_precomputed_guide_pattern: &ColorPattern, k_distances: &[f64], //weight by distance my_cost: &PrerenderedU8Function, guide_cost: Option<&PrerenderedU8Function>, @@ -1092,16 +1084,14 @@ fn find_best_match<'a>( .map(|d| d as f32) .collect(); - for (i, (candidate_pattern, candidate_guide_pattern)) in candidates_patterns - .iter() - .zip(candidates_guide_patterns.iter()) - .enumerate() - { + for (i, cand) in candidates.iter().enumerate() { if let Some(cost) = better_match( - &my_pattern, - candidate_pattern, - &my_guide_pattern, - candidate_guide_pattern, + &cand.k_neighs, + outside_color, + source_maps, + &guides, + &my_precomputed_pattern, + &my_precomputed_guide_pattern, distance_gaussians.as_slice(), my_cost, guide_cost, @@ -1117,10 +1107,12 @@ fn find_best_match<'a>( #[allow(clippy::too_many_arguments)] fn better_match( - my_pattern: &ColorPattern, - candidate_pattern: &ColorPattern, - my_guide_pattern: &ColorPattern, - candidate_guide_pattern: &ColorPattern, + k_neighs: &[(SignedCoord2D, MapId)], + outside_color: image::Rgba, + source_maps: &[ImageBuffer<'_>], + guides: &Option>, + my_precomputed_pattern: &ColorPattern, + my_precomputed_guide_pattern: &ColorPattern, distance_gaussians: &[f32], //weight by distance my_cost: &PrerenderedU8Function, guide_cost: Option<&PrerenderedU8Function>, @@ -1128,34 +1120,50 @@ fn better_match( ) -> Option { let mut score: f32 = 0.0; //minimize score - for ((my_value, candidate_value), dist_gaussian) in my_pattern - .0 - .iter() - .copied() - .zip(candidate_pattern.0.iter().copied()) - .zip(distance_gaussians.iter().copied()) - { - score += dist_gaussian * my_cost.get(my_value, candidate_value); - - if score >= current_best { - return None; - } - } + let mut i = 0; + let mut next_pixel = [0; 4]; + for (n_coord, n_map) in k_neighs { + let end = i + 4; - if let Some(guide_cost_fn) = guide_cost { - for ((my_guide, candidate_guide), dist_gaussian) in my_guide_pattern - .0 - .iter() - .copied() - .zip(candidate_guide_pattern.0.iter().copied()) - .zip(distance_gaussians.iter().copied()) - { - score += dist_gaussian * guide_cost_fn.get(my_guide, candidate_guide); + //check if he haven't gone outside the possible bounds + get_color_of_neighbor( + outside_color, + source_maps, + *n_coord, + *n_map, + &mut next_pixel, + false, + (0, 0), + ); + for j in 0..4 { + score += distance_gaussians[i + j] + * my_cost.get(my_precomputed_pattern.0[j + i], next_pixel[j]); if score >= current_best { return None; } } + + if let Some(guide_cost) = guide_cost { + let example_guides = &(guides.as_ref().unwrap().example_guides); + get_color_of_neighbor( + outside_color, + example_guides, + *n_coord, + *n_map, + &mut next_pixel, + false, + (0, 0), + ); + for j in 0..4 { + score += distance_gaussians[i + j] + * guide_cost.get(my_precomputed_guide_pattern.0[j + i], next_pixel[j]); + if score >= current_best { + return None; + } + } + } + i = end; } Some(score) From 7470223e42deeff8c427ee18dc868b68aaf5f8d3 Mon Sep 17 00:00:00 2001 From: Peter Stefek Date: Thu, 21 Nov 2019 21:21:10 -0800 Subject: [PATCH 2/7] removed in progress comment --- lib/src/multires_stochastic_texture_synthesis.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/src/multires_stochastic_texture_synthesis.rs b/lib/src/multires_stochastic_texture_synthesis.rs index aff80c8..f977e4f 100644 --- a/lib/src/multires_stochastic_texture_synthesis.rs +++ b/lib/src/multires_stochastic_texture_synthesis.rs @@ -889,7 +889,6 @@ impl Generator { is_tiling_mode, ); - // TODO Peter do not forget to move guide cost lookup table // 3.2 get pattern for guide map if we have them let (my_cost, guide_cost) = if let Some(ref in_guides) = guides { //get example pattern to compare to From c3eba95b3111f90f58c6ebe14ea12ca37df42bf0 Mon Sep 17 00:00:00 2001 From: Peter Stefek Date: Thu, 21 Nov 2019 23:14:29 -0800 Subject: [PATCH 3/7] optimized the optimization --- .../multires_stochastic_texture_synthesis.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/src/multires_stochastic_texture_synthesis.rs b/lib/src/multires_stochastic_texture_synthesis.rs index f977e4f..117c290 100644 --- a/lib/src/multires_stochastic_texture_synthesis.rs +++ b/lib/src/multires_stochastic_texture_synthesis.rs @@ -1121,6 +1121,7 @@ fn better_match( let mut i = 0; let mut next_pixel = [0; 4]; + let mut next_pixel_score: f32 = 0.0; for (n_coord, n_map) in k_neighs { let end = i + 4; @@ -1136,11 +1137,7 @@ fn better_match( ); for j in 0..4 { - score += distance_gaussians[i + j] - * my_cost.get(my_precomputed_pattern.0[j + i], next_pixel[j]); - if score >= current_best { - return None; - } + next_pixel_score = my_cost.get(my_precomputed_pattern.0[j + i], next_pixel[j]); } if let Some(guide_cost) = guide_cost { @@ -1155,13 +1152,14 @@ fn better_match( (0, 0), ); for j in 0..4 { - score += distance_gaussians[i + j] - * guide_cost.get(my_precomputed_guide_pattern.0[j + i], next_pixel[j]); - if score >= current_best { - return None; - } + next_pixel_score += + guide_cost.get(my_precomputed_guide_pattern.0[j + i], next_pixel[j]); } } + score += next_pixel_score * distance_gaussians[i]; + if score >= current_best { + return None; + } i = end; } From 622cdf5631af238165cf584f50282ec7048f6fe0 Mon Sep 17 00:00:00 2001 From: Peter Stefek Date: Thu, 21 Nov 2019 23:14:52 -0800 Subject: [PATCH 4/7] fmt --- lib/src/multires_stochastic_texture_synthesis.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/src/multires_stochastic_texture_synthesis.rs b/lib/src/multires_stochastic_texture_synthesis.rs index 117c290..93b1f3a 100644 --- a/lib/src/multires_stochastic_texture_synthesis.rs +++ b/lib/src/multires_stochastic_texture_synthesis.rs @@ -1121,8 +1121,9 @@ fn better_match( let mut i = 0; let mut next_pixel = [0; 4]; - let mut next_pixel_score: f32 = 0.0; + let mut next_pixel_score: f32; for (n_coord, n_map) in k_neighs { + next_pixel_score = 0.0; let end = i + 4; //check if he haven't gone outside the possible bounds @@ -1137,7 +1138,7 @@ fn better_match( ); for j in 0..4 { - next_pixel_score = my_cost.get(my_precomputed_pattern.0[j + i], next_pixel[j]); + next_pixel_score += my_cost.get(my_precomputed_pattern.0[j + i], next_pixel[j]); } if let Some(guide_cost) = guide_cost { From e9401c42223c1ee2cd2571fe78620f7265a65199 Mon Sep 17 00:00:00 2001 From: Peter Stefek Date: Thu, 21 Nov 2019 23:43:50 -0800 Subject: [PATCH 5/7] all hail clippy --- lib/src/multires_stochastic_texture_synthesis.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/src/multires_stochastic_texture_synthesis.rs b/lib/src/multires_stochastic_texture_synthesis.rs index 93b1f3a..da49f16 100644 --- a/lib/src/multires_stochastic_texture_synthesis.rs +++ b/lib/src/multires_stochastic_texture_synthesis.rs @@ -1137,8 +1137,8 @@ fn better_match( (0, 0), ); - for j in 0..4 { - next_pixel_score += my_cost.get(my_precomputed_pattern.0[j + i], next_pixel[j]); + for (channel_n, &channel) in next_pixel.iter().enumerate() { + next_pixel_score += my_cost.get(my_precomputed_pattern.0[i + channel_n], channel); } if let Some(guide_cost) = guide_cost { @@ -1152,9 +1152,10 @@ fn better_match( false, (0, 0), ); - for j in 0..4 { + + for (channel_n, &channel) in next_pixel.iter().enumerate() { next_pixel_score += - guide_cost.get(my_precomputed_guide_pattern.0[j + i], next_pixel[j]); + guide_cost.get(my_precomputed_guide_pattern.0[i + channel_n], channel); } } score += next_pixel_score * distance_gaussians[i]; From 9cb6f58c46a8d10599b5edc10af231f65798dc11 Mon Sep 17 00:00:00 2001 From: Peter Stefek Date: Fri, 22 Nov 2019 09:04:39 -0800 Subject: [PATCH 6/7] faster for guide map images --- .../multires_stochastic_texture_synthesis.rs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/src/multires_stochastic_texture_synthesis.rs b/lib/src/multires_stochastic_texture_synthesis.rs index da49f16..63a496a 100644 --- a/lib/src/multires_stochastic_texture_synthesis.rs +++ b/lib/src/multires_stochastic_texture_synthesis.rs @@ -1141,8 +1141,18 @@ fn better_match( next_pixel_score += my_cost.get(my_precomputed_pattern.0[i + channel_n], channel); } - if let Some(guide_cost) = guide_cost { - let example_guides = &(guides.as_ref().unwrap().example_guides); + score += next_pixel_score * distance_gaussians[i]; + if score >= current_best { + return None; + } + i = end; + } + + i = 0; + if let Some(guide_cost) = guide_cost { + let example_guides = &(guides.as_ref().unwrap().example_guides); + for (n_coord, n_map) in k_neighs { + next_pixel_score = 0.0; get_color_of_neighbor( outside_color, example_guides, @@ -1157,12 +1167,12 @@ fn better_match( next_pixel_score += guide_cost.get(my_precomputed_guide_pattern.0[i + channel_n], channel); } + score += next_pixel_score * distance_gaussians[i]; + if score >= current_best { + return None; + } + i += 4; } - score += next_pixel_score * distance_gaussians[i]; - if score >= current_best { - return None; - } - i = end; } Some(score) From bf78da07cfa42678a563515e2d27948b906e11da Mon Sep 17 00:00:00 2001 From: Peter Stefek Date: Fri, 22 Nov 2019 21:04:00 -0800 Subject: [PATCH 7/7] Revert "faster for guide map images" This reverts commit 9cb6f58c46a8d10599b5edc10af231f65798dc11. --- .../multires_stochastic_texture_synthesis.rs | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/lib/src/multires_stochastic_texture_synthesis.rs b/lib/src/multires_stochastic_texture_synthesis.rs index 63a496a..da49f16 100644 --- a/lib/src/multires_stochastic_texture_synthesis.rs +++ b/lib/src/multires_stochastic_texture_synthesis.rs @@ -1141,18 +1141,8 @@ fn better_match( next_pixel_score += my_cost.get(my_precomputed_pattern.0[i + channel_n], channel); } - score += next_pixel_score * distance_gaussians[i]; - if score >= current_best { - return None; - } - i = end; - } - - i = 0; - if let Some(guide_cost) = guide_cost { - let example_guides = &(guides.as_ref().unwrap().example_guides); - for (n_coord, n_map) in k_neighs { - next_pixel_score = 0.0; + if let Some(guide_cost) = guide_cost { + let example_guides = &(guides.as_ref().unwrap().example_guides); get_color_of_neighbor( outside_color, example_guides, @@ -1167,12 +1157,12 @@ fn better_match( next_pixel_score += guide_cost.get(my_precomputed_guide_pattern.0[i + channel_n], channel); } - score += next_pixel_score * distance_gaussians[i]; - if score >= current_best { - return None; - } - i += 4; } + score += next_pixel_score * distance_gaussians[i]; + if score >= current_best { + return None; + } + i = end; } Some(score)