From bd48c208e7eb69dfde5c9bd69d65a0f03dcf07ee Mon Sep 17 00:00:00 2001 From: Shaan Date: Tue, 13 Aug 2024 16:44:45 -0700 Subject: [PATCH 1/5] Added benchmark_dry_run boolean flag to command line options --- src/benchmark.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/benchmark.cc b/src/benchmark.cc index b7767bd00..54b7d02bb 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -92,6 +92,10 @@ BM_DEFINE_double(benchmark_min_warmup_time, 0.0); // standard deviation of the runs will be reported. BM_DEFINE_int32(benchmark_repetitions, 1); +// If enabled, forces each benchmark to execute exactly one iteration and one repetition, +// bypassing any configured MinTime()/MinWarmUpTime()/Iterations()/Repetitions() +BM_DEFINE_bool(benchmark_dry_run, false); + // If set, enable random interleaving of repetitions of all benchmarks. // See http://github.com/google/benchmark/issues/1051 for details. BM_DEFINE_bool(benchmark_enable_random_interleaving, false); @@ -717,6 +721,7 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_min_warmup_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || + ParseBoolFlag(argv[i], "benchmark_dry_run", &FLAGS_benchmark_dry_run) || ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", &FLAGS_benchmark_enable_random_interleaving) || ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", @@ -783,6 +788,7 @@ void PrintDefaultHelp() { " [--benchmark_min_time=`x` OR `s` ]\n" " [--benchmark_min_warmup_time=]\n" " [--benchmark_repetitions=]\n" + " [--benchmark_dry_run={true|false}]\n" " [--benchmark_enable_random_interleaving={true|false}]\n" " [--benchmark_report_aggregates_only={true|false}]\n" " [--benchmark_display_aggregates_only={true|false}]\n" From 31c75395c67c2770f955557075090e0a8411cc1b Mon Sep 17 00:00:00 2001 From: Shaan Date: Sat, 17 Aug 2024 14:30:55 -0700 Subject: [PATCH 2/5] Dry run logic to exit early and override iterations, repetitions, min time, min warmup time --- src/benchmark_runner.cc | 30 ++++++++++++++++++++---------- src/benchmark_runner.h | 1 + 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index a38093937..685f5c688 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -80,6 +80,10 @@ BenchmarkReporter::Run CreateRunReport( BenchmarkReporter::Run report; report.run_name = b.name(); + + if (FLAGS_benchmark_dry_run) { + report.run_name.function_name.append("/dry_run"); + } report.family_index = b.family_index(); report.per_family_instance_index = b.per_family_instance_index(); report.skipped = results.skipped_; @@ -228,20 +232,21 @@ BenchmarkRunner::BenchmarkRunner( : b(b_), reports_for_family(reports_for_family_), parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), - min_time(ComputeMinTime(b_, parsed_benchtime_flag)), - min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) - ? b.min_warmup_time() - : FLAGS_benchmark_min_warmup_time), + min_time(FLAGS_benchmark_dry_run ? 0 : ComputeMinTime(b_, parsed_benchtime_flag)), + min_warmup_time(FLAGS_benchmark_dry_run ? 0 + : (!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) ? b.min_warmup_time() + : FLAGS_benchmark_min_warmup_time), warmup_done(!(min_warmup_time > 0.0)), - repeats(b.repetitions() != 0 ? b.repetitions() - : FLAGS_benchmark_repetitions), + repeats(FLAGS_benchmark_dry_run ? 1 + : b.repetitions() != 0 ? b.repetitions() + : FLAGS_benchmark_repetitions), has_explicit_iteration_count(b.iterations() != 0 || parsed_benchtime_flag.tag == BenchTimeType::ITERS), pool(static_cast(b.threads() - 1)), - iters(has_explicit_iteration_count - ? ComputeIters(b_, parsed_benchtime_flag) - : 1), + iters(FLAGS_benchmark_dry_run ? 1 + : has_explicit_iteration_count ? ComputeIters(b_, parsed_benchtime_flag) + : 1), perf_counters_measurement_ptr(pcm_) { run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only || @@ -295,6 +300,10 @@ BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { BM_VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" << i.results.real_time_used << "\n"; + // In dry run mode, we report 1 iteration even if each thread runs a singular iteration. + if (FLAGS_benchmark_dry_run) { + i.results.iterations /= b.threads(); + } // By using KeepRunningBatch a benchmark can iterate more times than // requested, so take the iteration count from i.results. i.iters = i.results.iterations / b.threads(); @@ -339,7 +348,8 @@ bool BenchmarkRunner::ShouldReportIterationResults( // Determine if this run should be reported; // Either it has run for a sufficient amount of time // or because an error was reported. - return i.results.skipped_ || + return i.results.skipped_ || + FLAGS_benchmark_dry_run || i.iters >= kMaxIterations || // Too many iterations already. i.seconds >= GetMinTimeToApply() || // The elapsed time is large enough. diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index cd34d2d5b..25e3ddca3 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -28,6 +28,7 @@ namespace benchmark { BM_DECLARE_string(benchmark_min_time); BM_DECLARE_double(benchmark_min_warmup_time); BM_DECLARE_int32(benchmark_repetitions); +BM_DECLARE_bool(benchmark_dry_run); BM_DECLARE_bool(benchmark_report_aggregates_only); BM_DECLARE_bool(benchmark_display_aggregates_only); BM_DECLARE_string(benchmark_perf_counters); From 14450da3eb0962be510744dbba64ca72869e780d Mon Sep 17 00:00:00 2001 From: Shaan Date: Wed, 11 Sep 2024 19:57:14 -0700 Subject: [PATCH 3/5] Changeddry run override logic structure and added dry run to context --- src/benchmark.cc | 8 ++++++-- src/benchmark_runner.cc | 43 +++++++++++++++++++++-------------------- src/benchmark_runner.h | 6 +++--- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 54b7d02bb..260507744 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -92,8 +92,9 @@ BM_DEFINE_double(benchmark_min_warmup_time, 0.0); // standard deviation of the runs will be reported. BM_DEFINE_int32(benchmark_repetitions, 1); -// If enabled, forces each benchmark to execute exactly one iteration and one repetition, -// bypassing any configured MinTime()/MinWarmUpTime()/Iterations()/Repetitions() +// If enabled, forces each benchmark to execute exactly one iteration and one +// repetition, bypassing any configured +// MinTime()/MinWarmUpTime()/Iterations()/Repetitions() BM_DEFINE_bool(benchmark_dry_run, false); // If set, enable random interleaving of repetitions of all benchmarks. @@ -760,6 +761,9 @@ void ParseCommandLineFlags(int* argc, char** argv) { if (FLAGS_benchmark_color.empty()) { PrintUsageAndExit(); } + if (FLAGS_benchmark_dry_run) { + AddCustomContext("dry_run", "true"); + } for (const auto& kv : FLAGS_benchmark_context) { AddCustomContext(kv.first, kv.second); } diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 4045cd64f..a59b16ba6 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -80,10 +80,6 @@ BenchmarkReporter::Run CreateRunReport( BenchmarkReporter::Run report; report.run_name = b.name(); - - if (FLAGS_benchmark_dry_run) { - report.run_name.function_name.append("/dry_run"); - } report.family_index = b.family_index(); report.per_family_instance_index = b.per_family_instance_index(); report.skipped = results.skipped_; @@ -232,22 +228,32 @@ BenchmarkRunner::BenchmarkRunner( : b(b_), reports_for_family(reports_for_family_), parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), - min_time(FLAGS_benchmark_dry_run ? 0 : ComputeMinTime(b_, parsed_benchtime_flag)), - min_warmup_time(FLAGS_benchmark_dry_run ? 0 - : (!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) ? b.min_warmup_time() - : FLAGS_benchmark_min_warmup_time), - warmup_done(!(min_warmup_time > 0.0)), - repeats(FLAGS_benchmark_dry_run ? 1 - : b.repetitions() != 0 ? b.repetitions() - : FLAGS_benchmark_repetitions), has_explicit_iteration_count(b.iterations() != 0 || parsed_benchtime_flag.tag == BenchTimeType::ITERS), pool(static_cast(b.threads() - 1)), - iters(FLAGS_benchmark_dry_run ? 1 - : has_explicit_iteration_count ? ComputeIters(b_, parsed_benchtime_flag) - : 1), + iters(has_explicit_iteration_count + ? ComputeIters(b_, parsed_benchtime_flag) + : 1), perf_counters_measurement_ptr(pcm_) { + if (FLAGS_benchmark_dry_run) { + min_time = 0; + min_warmup_time = 0; + warmup_done = true; + repeats = 1; + iters = 1; + } else { + min_time = ComputeMinTime(b_, parsed_benchtime_flag); + min_warmup_time = ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) + ? b.min_warmup_time() + : FLAGS_benchmark_min_warmup_time); + warmup_done = !(min_warmup_time > 0.0); + repeats = + b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions; + iters = has_explicit_iteration_count + ? ComputeIters(b_, parsed_benchtime_flag) + : 1; + } run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only || FLAGS_benchmark_display_aggregates_only); @@ -300,10 +306,6 @@ BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { BM_VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" << i.results.real_time_used << "\n"; - // In dry run mode, we report 1 iteration even if each thread runs a singular iteration. - if (FLAGS_benchmark_dry_run) { - i.results.iterations /= b.threads(); - } // By using KeepRunningBatch a benchmark can iterate more times than // requested, so take the iteration count from i.results. i.iters = i.results.iterations / b.threads(); @@ -348,8 +350,7 @@ bool BenchmarkRunner::ShouldReportIterationResults( // Determine if this run should be reported; // Either it has run for a sufficient amount of time // or because an error was reported. - return i.results.skipped_ || - FLAGS_benchmark_dry_run || + return i.results.skipped_ || FLAGS_benchmark_dry_run || i.iters >= kMaxIterations || // Too many iterations already. i.seconds >= GetMinTimeToApply() || // The elapsed time is large enough. diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 25e3ddca3..10bc51386 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -90,10 +90,10 @@ class BenchmarkRunner { BenchmarkReporter::PerFamilyRunReports* reports_for_family; BenchTimeType parsed_benchtime_flag; - const double min_time; - const double min_warmup_time; + double min_time; + double min_warmup_time; bool warmup_done; - const int repeats; + int repeats; const bool has_explicit_iteration_count; int num_repetitions_done = 0; From d663b9b7236223b523d1271aaeea169f136feb52 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 12 Sep 2024 11:56:54 +0100 Subject: [PATCH 4/5] cleanups --- src/benchmark_runner.cc | 37 +++++++++++++++++-------------------- src/benchmark_runner.h | 14 +++----------- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index a59b16ba6..524f8a1ce 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -58,6 +58,14 @@ namespace benchmark { +BM_DECLARE_bool(benchmark_dry_run); +BM_DECLARE_string(benchmark_min_time); +BM_DECLARE_double(benchmark_min_warmup_time); +BM_DECLARE_int32(benchmark_repetitions); +BM_DECLARE_bool(benchmark_report_aggregates_only); +BM_DECLARE_bool(benchmark_display_aggregates_only); +BM_DECLARE_string(benchmark_perf_counters); + namespace internal { MemoryManager* memory_manager = nullptr; @@ -228,32 +236,21 @@ BenchmarkRunner::BenchmarkRunner( : b(b_), reports_for_family(reports_for_family_), parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), + min_time(FLAGS_benchmark_dry_run ? 0 : ComputeMinTime(b_, parsed_benchtime_flag)), + min_warmup_time(FLAGS_benchmark_dry_run ? 0 : ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) + ? b.min_warmup_time() + : FLAGS_benchmark_min_warmup_time)), + warmup_done(FLAGS_benchmark_dry_run ? true : !(min_warmup_time > 0.0)), + repeats(FLAGS_benchmark_dry_run ? 1 : (b.repetitions() != 0 ? b.repetitions() + : FLAGS_benchmark_repetitions)), has_explicit_iteration_count(b.iterations() != 0 || parsed_benchtime_flag.tag == BenchTimeType::ITERS), pool(static_cast(b.threads() - 1)), - iters(has_explicit_iteration_count + iters(FLAGS_benchmark_dry_run ? 1 : (has_explicit_iteration_count ? ComputeIters(b_, parsed_benchtime_flag) - : 1), + : 1)), perf_counters_measurement_ptr(pcm_) { - if (FLAGS_benchmark_dry_run) { - min_time = 0; - min_warmup_time = 0; - warmup_done = true; - repeats = 1; - iters = 1; - } else { - min_time = ComputeMinTime(b_, parsed_benchtime_flag); - min_warmup_time = ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) - ? b.min_warmup_time() - : FLAGS_benchmark_min_warmup_time); - warmup_done = !(min_warmup_time > 0.0); - repeats = - b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions; - iters = has_explicit_iteration_count - ? ComputeIters(b_, parsed_benchtime_flag) - : 1; - } run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only || FLAGS_benchmark_display_aggregates_only); diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 10bc51386..6e5ceb31e 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -25,14 +25,6 @@ namespace benchmark { -BM_DECLARE_string(benchmark_min_time); -BM_DECLARE_double(benchmark_min_warmup_time); -BM_DECLARE_int32(benchmark_repetitions); -BM_DECLARE_bool(benchmark_dry_run); -BM_DECLARE_bool(benchmark_report_aggregates_only); -BM_DECLARE_bool(benchmark_display_aggregates_only); -BM_DECLARE_string(benchmark_perf_counters); - namespace internal { extern MemoryManager* memory_manager; @@ -90,10 +82,10 @@ class BenchmarkRunner { BenchmarkReporter::PerFamilyRunReports* reports_for_family; BenchTimeType parsed_benchtime_flag; - double min_time; - double min_warmup_time; + const double min_time; + const double min_warmup_time; bool warmup_done; - int repeats; + const int repeats; const bool has_explicit_iteration_count; int num_repetitions_done = 0; From 59b6daf4c173b86bcfa16a2ebca6d764201e7a34 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 12 Sep 2024 11:58:19 +0100 Subject: [PATCH 5/5] clang-format --- src/benchmark_runner.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 524f8a1ce..463f69fc5 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -236,20 +236,29 @@ BenchmarkRunner::BenchmarkRunner( : b(b_), reports_for_family(reports_for_family_), parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), - min_time(FLAGS_benchmark_dry_run ? 0 : ComputeMinTime(b_, parsed_benchtime_flag)), - min_warmup_time(FLAGS_benchmark_dry_run ? 0 : ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) - ? b.min_warmup_time() - : FLAGS_benchmark_min_warmup_time)), + min_time(FLAGS_benchmark_dry_run + ? 0 + : ComputeMinTime(b_, parsed_benchtime_flag)), + min_warmup_time( + FLAGS_benchmark_dry_run + ? 0 + : ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) + ? b.min_warmup_time() + : FLAGS_benchmark_min_warmup_time)), warmup_done(FLAGS_benchmark_dry_run ? true : !(min_warmup_time > 0.0)), - repeats(FLAGS_benchmark_dry_run ? 1 : (b.repetitions() != 0 ? b.repetitions() - : FLAGS_benchmark_repetitions)), + repeats(FLAGS_benchmark_dry_run + ? 1 + : (b.repetitions() != 0 ? b.repetitions() + : FLAGS_benchmark_repetitions)), has_explicit_iteration_count(b.iterations() != 0 || parsed_benchtime_flag.tag == BenchTimeType::ITERS), pool(static_cast(b.threads() - 1)), - iters(FLAGS_benchmark_dry_run ? 1 : (has_explicit_iteration_count - ? ComputeIters(b_, parsed_benchtime_flag) - : 1)), + iters(FLAGS_benchmark_dry_run + ? 1 + : (has_explicit_iteration_count + ? ComputeIters(b_, parsed_benchtime_flag) + : 1)), perf_counters_measurement_ptr(pcm_) { run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only ||