Skip to content

Commit

Permalink
Fix benches for ia_loop, circles (#1922)
Browse files Browse the repository at this point in the history
  • Loading branch information
mamcx authored Nov 5, 2024
1 parent e94009a commit cb9e017
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 30 deletions.
43 changes: 19 additions & 24 deletions crates/bench/benches/special.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use spacetimedb_bench::{
use spacetimedb_lib::{bsatn::ToBsatn as _, sats, ProductValue};
use spacetimedb_schema::schema::TableSchema;
use spacetimedb_testing::modules::start_runtime;
use std::time::{Duration, Instant};
use std::sync::OnceLock;
use std::{hint::black_box, sync::Arc};

#[global_allocator]
Expand Down Expand Up @@ -62,53 +62,48 @@ fn custom_db_benchmarks(c: &mut Criterion) {
let module = runtime.block_on(async { BENCHMARKS_MODULE.load_module(config, None).await });
let mut group = c.benchmark_group("special");
// This bench take long, so adjust for it
group.measurement_time(Duration::from_secs(60 * 2));
group.sample_size(10);

let init_db: OnceLock<()> = OnceLock::new();
for n in [10, 100] {
let args = sats::product![n];
group.bench_function(&format!("db_game/circles/load={n}"), |b| {
b.iter_custom(|iters| {
// Initialize outside the benchmark so the db is seed once, to avoid to enlarge the db
init_db.get_or_init(|| {
runtime.block_on(async {
module
.call_reducer_binary("init_game_circles", args.clone())
.call_reducer_binary("init_game_circles", sats::product![100])
.await
.unwrap()
});
});

let start = Instant::now();

for _ in 0..iters {
black_box(
runtime.block_on(async {
module.call_reducer_binary("run_game_circles", args.clone()).await.ok()
}),
);
}

start.elapsed()
b.iter(|| {
black_box(
runtime.block_on(async { module.call_reducer_binary("run_game_circles", args.clone()).await.ok() }),
);
})
});
}

for n in [10, 100] {
let init_db: OnceLock<()> = OnceLock::new();
for n in [500, 5_000] {
let args = sats::product![n];
group.bench_function(&format!("db_game/ia_loop/load={n}"), |b| {
b.iter_custom(|_iters| {
// Initialize outside the benchmark so the db is seed once, to avoid `unique` constraints violations
init_db.get_or_init(|| {
runtime.block_on(async {
module
.call_reducer_binary("init_game_ia_loop", args.clone())
.call_reducer_binary("init_game_ia_loop", sats::product![5_000])
.await
.unwrap()
});

let start = Instant::now();
.unwrap();
})
});

b.iter(|| {
black_box(
runtime.block_on(async { module.call_reducer_binary("run_game_ia_loop", args.clone()).await.ok() }),
);

start.elapsed()
})
});
}
Expand Down
2 changes: 1 addition & 1 deletion crates/testing/tests/standalone_integration_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ fn test_calling_bench_db_ia_loop() {
("update_position_all", 20_000, "UPDATE POSITION ALL: 20000, processed: 20000"),
("update_position_with_velocity", 10_000, "UPDATE POSITION BY VELOCITY: 10000, processed: 10000"),
("insert_world", 5_000, "INSERT WORLD PLAYERS: 5000"),
("game_loop_enemy_ia", 5_000, "ENEMY IA LOOP PLAYERS: 5000, processed: 2500"),
("game_loop_enemy_ia", 5_000, "ENEMY IA LOOP PLAYERS: 5000, processed: 5000"),
];

_run_bench_db(module, &benches).await
Expand Down
6 changes: 1 addition & 5 deletions modules/benchmarks/src/ia_loop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ fn move_agent(
entity_id,
location_x: mobile_entity.location_x + 1,
location_y: mobile_entity.location_y + 1,
timestamp: moment_milliseconds(),
timestamp: agent.next_action_timestamp,
};

ctx.db.game_enemy_ai_agent_state().entity_id().update(agent.clone());
Expand Down Expand Up @@ -394,10 +394,6 @@ pub fn game_loop_enemy_ia(ctx: &ReducerContext, players: u64) {
let current_time_ms = moment_milliseconds();

for mut agent in ctx.db.game_enemy_ai_agent_state().iter() {
if agent.next_action_timestamp > current_time_ms {
continue;
}

let agent_targetable = ctx
.db
.game_targetable_state()
Expand Down

2 comments on commit cb9e017

@github-actions
Copy link

@github-actions github-actions bot commented on cb9e017 Nov 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmarking failed. Please check the workflow run for details.

@github-actions
Copy link

@github-actions github-actions bot commented on cb9e017 Nov 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Callgrind benchmark results

Callgrind Benchmark Report

These benchmarks were run using callgrind,
an instruction-level profiler. They allow comparisons between sqlite (sqlite), SpacetimeDB running through a module (stdb_module), and the underlying SpacetimeDB data storage engine (stdb_raw). Callgrind emulates a CPU to collect the below estimates.

Measurement changes larger than five percent are in bold.

In-memory benchmarks

callgrind: empty transaction

db total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw 6364 6364 0.00% 6460 6460 0.00%
sqlite 5579 5579 0.00% 5965 5965 0.00%

callgrind: filter

db schema indices count preload _column data_type total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str no_index 64 128 1 u64 76558 76558 0.00% 77062 77058 0.01%
stdb_raw u32_u64_str no_index 64 128 2 string 119056 119056 0.00% 119786 119858 -0.06%
stdb_raw u32_u64_str btree_each_column 64 128 2 string 25048 25052 -0.02% 25658 25666 -0.03%
stdb_raw u32_u64_str btree_each_column 64 128 1 u64 24016 24016 0.00% 24424 24432 -0.03%
sqlite u32_u64_str no_index 64 128 2 string 144695 144695 0.00% 146107 146115 -0.01%
sqlite u32_u64_str no_index 64 128 1 u64 124044 124050 -0.00% 125262 125268 -0.00%
sqlite u32_u64_str btree_each_column 64 128 1 u64 131361 131361 0.00% 132799 132803 -0.00%
sqlite u32_u64_str btree_each_column 64 128 2 string 134494 134494 0.00% 136182 136178 0.00%

callgrind: insert bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 64 128 878435 877856 0.07% 934721 903210 3.49%
stdb_raw u32_u64_str btree_each_column 64 128 1029694 1028456 0.12% 1063298 1093252 -2.74%
sqlite u32_u64_str unique_0 64 128 398320 398320 0.00% 414694 414690 0.00%
sqlite u32_u64_str btree_each_column 64 128 983637 983637 0.00% 1016423 1016423 0.00%

callgrind: iterate

db schema indices count total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 153691 153691 0.00% 153801 153801 0.00%
stdb_raw u32_u64_str unique_0 64 16716 16716 0.00% 16806 16830 -0.14%
sqlite u32_u64_str unique_0 1024 1067255 1067255 0.00% 1070719 1070719 0.00%
sqlite u32_u64_str unique_0 64 76201 76201 0.00% 77315 77315 0.00%

callgrind: serialize_product_value

count format total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
64 json 47528 47528 0.00% 50282 50282 0.00%
64 bsatn 25509 25509 0.00% 27753 27753 0.00%
16 bsatn 8200 8200 0.00% 9560 9560 0.00%
16 json 12188 12188 0.00% 14194 14194 0.00%

callgrind: update bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 1024 20115267 20519049 -1.97% 20743453 21186229 -2.09%
stdb_raw u32_u64_str unique_0 64 128 1286470 1287041 -0.04% 1362154 1332669 2.21%
sqlite u32_u64_str unique_0 1024 1024 1802200 1802182 0.00% 1811476 1811454 0.00%
sqlite u32_u64_str unique_0 64 128 128528 128528 0.00% 131498 131498 0.00%
On-disk benchmarks

callgrind: empty transaction

db total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw 6369 6369 0.00% 6473 6473 0.00%
sqlite 5621 5621 0.00% 6059 6059 0.00%

callgrind: filter

db schema indices count preload _column data_type total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str no_index 64 128 1 u64 76563 76563 0.00% 77035 77031 0.01%
stdb_raw u32_u64_str no_index 64 128 2 string 119061 119061 0.00% 119719 119859 -0.12%
stdb_raw u32_u64_str btree_each_column 64 128 2 string 25054 25053 0.00% 25608 25575 0.13%
stdb_raw u32_u64_str btree_each_column 64 128 1 u64 24021 24021 0.00% 24397 24401 -0.02%
sqlite u32_u64_str no_index 64 128 1 u64 125965 125965 0.00% 127403 127403 0.00%
sqlite u32_u64_str no_index 64 128 2 string 146616 146616 0.00% 148288 148288 0.00%
sqlite u32_u64_str btree_each_column 64 128 2 string 136616 136616 0.00% 138718 138714 0.00%
sqlite u32_u64_str btree_each_column 64 128 1 u64 133457 133457 0.00% 135353 135349 0.00%

callgrind: insert bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 64 128 826891 826707 0.02% 851317 851253 0.01%
stdb_raw u32_u64_str btree_each_column 64 128 977795 978716 -0.09% 1041793 1042124 -0.03%
sqlite u32_u64_str unique_0 64 128 415857 415857 0.00% 431637 431645 -0.00%
sqlite u32_u64_str btree_each_column 64 128 1021898 1021898 0.00% 1054020 1054012 0.00%

callgrind: iterate

db schema indices count total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 153696 153696 0.00% 153786 153782 0.00%
stdb_raw u32_u64_str unique_0 64 16721 16721 0.00% 16807 16807 0.00%
sqlite u32_u64_str unique_0 1024 1070323 1070323 0.00% 1074085 1074085 0.00%
sqlite u32_u64_str unique_0 64 77979 77973 0.01% 79309 79303 0.01%

callgrind: serialize_product_value

count format total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
64 json 47528 47528 0.00% 50282 50282 0.00%
64 bsatn 25509 25509 0.00% 27753 27753 0.00%
16 bsatn 8200 8200 0.00% 9560 9560 0.00%
16 json 12188 12188 0.00% 14194 14194 0.00%

callgrind: update bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 1024 19037989 19036556 0.01% 19749951 19739060 0.06%
stdb_raw u32_u64_str unique_0 64 128 1239225 1240282 -0.09% 1313381 1314914 -0.12%
sqlite u32_u64_str unique_0 1024 1024 1809743 1809743 0.00% 1818367 1818367 0.00%
sqlite u32_u64_str unique_0 64 128 132654 132654 0.00% 135668 135668 0.00%

Please sign in to comment.