Enabled verification with inconsistent number of batches (#51)

Author: Egor-Krivov

dl_bench/cli/launcher.py

@@ -17,6 +17,19 @@
 }
 
 
+def fix_lengths(outputs, ref_outputs):
+    """To speed up benchmarking we pass different number of batches for different backends.
+    Need to match the lenghts."""
+    min_lengths = min(len(outputs), len(ref_outputs))
+    if len(outputs) != len(ref_outputs):
+        print(
+            f"Slicing passed batches to smallest size {len(outputs)}->{min_lengths}; {len(ref_outputs)}->{min_lengths}"
+        )
+        return outputs[:min_lengths], ref_outputs[:min_lengths]
+    else:
+        return outputs, ref_outputs
+
+
 def parse_args():
     parser = argparse.ArgumentParser()
     # Benchmark
@@ -142,6 +155,7 @@ def main():
         reference_backend = Backend(device=ref_device, compiler="torch", dtype=dtype)
         _, ref_outputs = benchmark.inference(reference_backend)
         results, outputs = benchmark.inference(backend)
+        outputs, ref_outputs = fix_lengths(outputs, ref_outputs)
         cmp_res = compare(outputs, ref_outputs)
 
     print(f"Benchmark {benchmark_name} completed")