diff --git a/cpp/src/arrow/acero/aggregate_benchmark.cc b/cpp/src/arrow/acero/aggregate_benchmark.cc
index 654e037b2f886..0462d1ea860f6 100644
--- a/cpp/src/arrow/acero/aggregate_benchmark.cc
+++ b/cpp/src/arrow/acero/aggregate_benchmark.cc
@@ -926,7 +926,7 @@ static void BenchmarkRowSegmenter(benchmark::State& state, Args&&...) {
 
 std::vector<std::string> row_segmenter_argnames = {"Rows", "Segments", "SegmentKeys"};
 std::vector<std::vector<int64_t>> row_segmenter_args = {
-    {1 * 1024 * 1024}, benchmark::CreateRange(1, 4096, 16), {0, 1, 2, 3}};
+    {32 * 1024}, benchmark::CreateRange(1, 256, 4), benchmark::CreateDenseRange(0, 3, 1)};
 
 BENCHMARK(BenchmarkRowSegmenter)
     ->ArgNames(row_segmenter_argnames)