Add tests and benchmarks

.ci/bench.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+
+name=$(basename "$0")
+root=$(realpath "$(dirname "$(dirname "$0")")")
+target_dir="$root/lib/target"
+
+if ! [ -x "$(command -v critcmp)" ]; then
+    echo "Installing critcmp..."
+    cargo install critcmp
+fi
+
+function sub_help() {
+    printf "Usage: %s <subcommand> [options]\n" "$name"
+    echo "Subcommands:"
+    echo "    export    Exports the specified baselines to json in the benches dir"
+    echo "    cmp       Compare two benchmarks by name"
+    echo ""
+    echo "For help with each subcommand run:"
+    echo "$name <subcommand> -h | --help"
+    echo ""
+}
+
+function sub_cmp() {
+    critcmp --target-dir "$target_dir" "$@"
+}
+
+function sub_export() {
+    critcmp --target-dir "$target_dir" --export "$1" > "$root/lib/benches/${2:-$1}.json"
+}
+
+subcommand=$1
+case $subcommand in
+    "" | "-h" | "--help")
+        sub_help
+        ;;
+    *)
+        shift
+        "sub_${subcommand}" "$@"
+        if [ $? = 127 ]; then
+            echo "Error: '$subcommand' is not a known subcommand." >&2
+            echo "       Run '$name --help' for a list of known subcommands." >&2
+            exit 1
+        fi
+        ;;
+esac

.gitignore

@@ -1,5 +1,6 @@
 /target
 cli/target
+lib/target
 **/*.rs.bk
 Cargo.lock
 /out

CHANGELOG.md

@@ -5,6 +5,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- You can now specify the maximum number of threads that can be used at any one time
+via `SessionBuilder::max_thread_count`
+- CLI: You can now specify the maximum thread count via `-t | --threads`
+- Added `From<image::DynamicImage>` for `ImageSource`
+- Added integrations tests for different examples, to catch regressions in generation
+- Added criterion benchmarks for the different examples, to catch performance regressions
+
+### Changed
+- `SampleMethod::From<AsRef<Path>>` is now `SampleMethod::From<Into<ImageSource>>`
+- `Example::From<AsRef<Path>>` is now `Example::From<Into<ImageSource>>`
+
 ### Fixed
 - Disabled unused `rand` default features (OS random number generator)
 

Cargo.toml

@@ -3,3 +3,9 @@ members = [
     "lib",
     "cli",
 ]
+
+[patch.crates-io]
+# https://github.com/abonander/img_hash/pull/30
+# * Updates to use image 0.22
+# * Fixes a bug with to/from_base64
+img_hash = { git = "https://github.com/EmbarkStudios/img_hash.git", rev = "c40da78" }

cli/Cargo.toml

@@ -19,14 +19,15 @@ keywords = ["texture", "synthesis", "procedural", "cli"]
 [[bin]]
 name = "texture-synthesis"
 path = "src/main.rs"
+bench = false
 
 [dependencies]
 # We unfortunately can't use clicolors-control which is used by indicatif
 # because it only ever operates on stdout, even though we only ever print
 # to stderr. This is also why indicatif colors don't work if you pipe
 # the image output :(
 atty = "0.2.13"
-indicatif = "0.11.0"
+indicatif = "0.12.0"
 minifb = { version = "0.13.0", optional = true }
 structopt = "0.3.0"
 texture-synthesis = { version = "0.4.2", path = "../lib" }

cli/src/main.rs

@@ -165,6 +165,10 @@ struct Opt {
     /// * `uncertainty.png` - Map of pixels the generator was uncertain of
     #[structopt(long, parse(from_os_str))]
     debug_out_dir: Option<PathBuf>,
+    /// The maximum number of worker threads that can be active at any one time
+    /// while synthesizing images. Defaults to the logical core count.
+    #[structopt(short = "t", long = "threads")]
+    max_threads: Option<usize>,
     #[structopt(flatten)]
     tweaks: Tweaks,
     #[structopt(subcommand)]
@@ -255,9 +259,14 @@ fn real_main() -> Result<(), Error> {
         .guide_alpha(args.tweaks.alpha)
         .tiling_mode(args.tweaks.enable_tiling);
 
+    if let Some(mt) = args.max_threads {
+        sb = sb.max_thread_count(mt);
+    }
+
     if let Some(ref tg) = target_guide {
         sb = sb.load_target_guide(tg);
     }
+
     if let Some(rand_init) = args.tweaks.rand_init {
         sb = sb.random_init(rand_init);
     }

lib/Cargo.toml

@@ -33,6 +33,17 @@ default-features = false
 features = []
 
 [dependencies.image]
-version = "0.22.1"
+version = "0.22.2"
 default-features = false
 features = ["jpeg", "png_codec", "bmp"]
+
+[dev-dependencies]
+criterion = "0.3.0"
+img_hash = { version = "2.1.0", features = ["rust-image"] }
+
+[lib]
+bench = false
+
+[[bench]]
+name = "all-the-things"
+harness = false

lib/benches/all-the-things.rs

@@ -0,0 +1,239 @@
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use std::time::{Duration, Instant};
+use texture_synthesis as ts;
+
+fn single_example(c: &mut Criterion) {
+    static DIM: u32 = 25;
+
+    // Load the example image once to reduce variation between runs,
+    // though we still do a memcpy each run
+    let example_img = ts::image::open("../imgs/1.jpg").unwrap();
+
+    let mut group = c.benchmark_group("single_example");
+    group.sample_size(10);
+
+    for dim in [DIM, 2 * DIM, 4 * DIM, 8 * DIM, 16 * DIM].iter() {
+        group.bench_with_input(BenchmarkId::from_parameter(dim), dim, |b, &dim| {
+            b.iter_custom(|iters| {
+                let mut total_elapsed = Duration::new(0, 0);
+                for _i in 0..iters {
+                    let sess = ts::Session::builder()
+                        .add_example(example_img.clone())
+                        .seed(120)
+                        .output_size(dim, dim)
+                        .build()
+                        .unwrap();
+
+                    let start = Instant::now();
+                    black_box(sess.run(None));
+                    total_elapsed += start.elapsed();
+                }
+
+                total_elapsed
+            });
+        });
+    }
+    group.finish();
+}
+
+fn multi_example(c: &mut Criterion) {
+    static DIM: u32 = 25;
+
+    // Load the example image once to reduce variation between runs,
+    // though we still do a memcpy each run
+    let example_imgs = [
+        ts::image::open("../imgs/multiexample/1.jpg").unwrap(),
+        ts::image::open("../imgs/multiexample/2.jpg").unwrap(),
+        ts::image::open("../imgs/multiexample/3.jpg").unwrap(),
+        ts::image::open("../imgs/multiexample/4.jpg").unwrap(),
+    ];
+
+    let mut group = c.benchmark_group("multi_example");
+    group.sample_size(10);
+
+    for dim in [DIM, 2 * DIM, 4 * DIM, 8 * DIM, 16 * DIM].iter() {
+        group.bench_with_input(BenchmarkId::from_parameter(dim), dim, |b, &dim| {
+            b.iter_custom(|iters| {
+                let mut total_elapsed = Duration::new(0, 0);
+                for _i in 0..iters {
+                    let sess = ts::Session::builder()
+                        .add_examples(example_imgs.iter().cloned())
+                        .resize_input(dim, dim)
+                        //.random_init(10)
+                        .seed(211)
+                        .output_size(dim, dim)
+                        .build()
+                        .unwrap();
+
+                    let start = Instant::now();
+                    black_box(sess.run(None));
+                    total_elapsed += start.elapsed();
+                }
+
+                total_elapsed
+            });
+        });
+    }
+    group.finish();
+}
+
+fn guided(c: &mut Criterion) {
+    static DIM: u32 = 25;
+
+    // Load the example image once to reduce variation between runs,
+    // though we still do a memcpy each run
+    let example_img = ts::image::open("../imgs/2.jpg").unwrap();
+    let guide_img = ts::image::open("../imgs/masks/2_example.jpg").unwrap();
+    let target_img = ts::image::open("../imgs/masks/2_target.jpg").unwrap();
+
+    let mut group = c.benchmark_group("guided");
+    group.sample_size(10);
+
+    for dim in [DIM, 2 * DIM, 4 * DIM, 8 * DIM, 16 * DIM].iter() {
+        group.bench_with_input(BenchmarkId::from_parameter(dim), dim, |b, &dim| {
+            b.iter_custom(|iters| {
+                let mut total_elapsed = Duration::new(0, 0);
+                for _i in 0..iters {
+                    let sess = ts::Session::builder()
+                        .add_example(
+                            ts::Example::builder(example_img.clone()).with_guide(guide_img.clone()),
+                        )
+                        .load_target_guide(target_img.clone())
+                        //.random_init(10)
+                        .seed(211)
+                        .output_size(dim, dim)
+                        .build()
+                        .unwrap();
+
+                    let start = Instant::now();
+                    black_box(sess.run(None));
+                    total_elapsed += start.elapsed();
+                }
+
+                total_elapsed
+            });
+        });
+    }
+    group.finish();
+}
+
+fn style_transfer(c: &mut Criterion) {
+    static DIM: u32 = 25;
+
+    // Load the example image once to reduce variation between runs,
+    // though we still do a memcpy each run
+    let example_img = ts::image::open("../imgs/multiexample/4.jpg").unwrap();
+    let target_img = ts::image::open("../imgs/tom.jpg").unwrap();
+
+    let mut group = c.benchmark_group("style_transfer");
+    group.sample_size(10);
+
+    for dim in [DIM, 2 * DIM, 4 * DIM, 8 * DIM, 16 * DIM].iter() {
+        group.bench_with_input(BenchmarkId::from_parameter(dim), dim, |b, &dim| {
+            b.iter_custom(|iters| {
+                let mut total_elapsed = Duration::new(0, 0);
+                for _i in 0..iters {
+                    let sess = ts::Session::builder()
+                        .add_example(example_img.clone())
+                        .load_target_guide(target_img.clone())
+                        .output_size(dim, dim)
+                        .build()
+                        .unwrap();
+
+                    let start = Instant::now();
+                    black_box(sess.run(None));
+                    total_elapsed += start.elapsed();
+                }
+
+                total_elapsed
+            });
+        });
+    }
+    group.finish();
+}
+
+fn inpaint(c: &mut Criterion) {
+    static DIM: u32 = 25;
+
+    // Load the example image once to reduce variation between runs,
+    // though we still do a memcpy each run
+    let example_img = ts::image::open("../imgs/3.jpg").unwrap();
+    let inpaint_mask = ts::image::open("../imgs/masks/3_inpaint.jpg").unwrap();
+
+    let mut group = c.benchmark_group("inpaint");
+    group.sample_size(10);
+
+    for dim in [DIM, 2 * DIM, 4 * DIM, 8 * DIM, 16 * DIM].iter() {
+        group.bench_with_input(BenchmarkId::from_parameter(dim), dim, |b, &dim| {
+            b.iter_custom(|iters| {
+                let mut total_elapsed = Duration::new(0, 0);
+                for _i in 0..iters {
+                    let sess = ts::Session::builder()
+                        .inpaint_example(
+                            inpaint_mask.clone(),
+                            ts::Example::builder(example_img.clone())
+                                .set_sample_method(inpaint_mask.clone()),
+                        )
+                        .resize_input(dim, dim)
+                        .output_size(dim, dim)
+                        .build()
+                        .unwrap();
+
+                    let start = Instant::now();
+                    black_box(sess.run(None));
+                    total_elapsed += start.elapsed();
+                }
+
+                total_elapsed
+            });
+        });
+    }
+    group.finish();
+}
+
+fn tiling(c: &mut Criterion) {
+    static DIM: u32 = 25;
+
+    // Load the example image once to reduce variation between runs,
+    // though we still do a memcpy each run
+    let example_img = ts::image::open("../imgs/1.jpg").unwrap();
+    let inpaint_mask = ts::image::open("../imgs/masks/1_tile.jpg").unwrap();
+
+    let mut group = c.benchmark_group("tiling");
+    group.sample_size(10);
+
+    for dim in [DIM, 2 * DIM, 4 * DIM, 8 * DIM, 16 * DIM].iter() {
+        group.bench_with_input(BenchmarkId::from_parameter(dim), dim, |b, &dim| {
+            b.iter_custom(|iters| {
+                let mut total_elapsed = Duration::new(0, 0);
+                for _i in 0..iters {
+                    let sess = ts::Session::builder()
+                        .inpaint_example(inpaint_mask.clone(), example_img.clone())
+                        .resize_input(dim, dim)
+                        .output_size(dim, dim)
+                        .tiling_mode(true)
+                        .build()
+                        .unwrap();
+
+                    let start = Instant::now();
+                    black_box(sess.run(None));
+                    total_elapsed += start.elapsed();
+                }
+
+                total_elapsed
+            });
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    single_example,
+    multi_example,
+    guided,
+    style_transfer,
+    inpaint,
+    tiling,
+);
+criterion_main!(benches);