Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix benchmark flakiness #16649

Merged
merged 5 commits into from
Oct 24, 2023
Merged

Fix benchmark flakiness #16649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fe66ff9
Fix benchmark flakiness
fchirica Oct 17, 2023
8336c4c
Update test_offer_parsing_performance.py
fchirica Oct 18, 2023
76488fd
Merge branch 'main' into fc.bench
fchirica Oct 18, 2023
708e24e
Remove profilers.
fchirica Oct 20, 2023
755190d
Further remove profilers.
fchirica Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 1 addition & 19 deletions tests/core/full_node/test_performance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# flake8: noqa: F811, F401
from __future__ import annotations

import cProfile
import dataclasses
import logging
import random
Expand Down Expand Up @@ -116,10 +115,7 @@ async def test_full_block_performance(
spend_bundles.append(spend_bundle)
spend_bundle_ids.append(spend_bundle.get_hash())

pr = cProfile.Profile()
pr.enable()

with benchmark_runner.assert_runtime(seconds=0.001, label=f"{request.node.name} - mempool"):
with benchmark_runner.assert_runtime(seconds=0.0055, label=f"{request.node.name} - mempool"):
num_tx: int = 0
for spend_bundle, spend_bundle_id in zip(spend_bundles, spend_bundle_ids):
num_tx += 1
Expand All @@ -134,8 +130,6 @@ async def test_full_block_performance(
break

log.warning(f"Num Tx: {num_tx}")
pr.create_stats()
pr.dump_stats("./mempool-benchmark.pstats")

# Create an unfinished block
peak = full_node_1.full_node.blockchain.get_peak()
Expand Down Expand Up @@ -173,26 +167,14 @@ async def test_full_block_performance(
[],
)

pr = cProfile.Profile()
pr.enable()

with benchmark_runner.assert_runtime(seconds=0.1, label=f"{request.node.name} - unfinished"):
res = await full_node_1.respond_unfinished_block(fnp.RespondUnfinishedBlock(unfinished), fake_peer)

log.warning(f"Res: {res}")

pr.create_stats()
pr.dump_stats("./unfinished-benchmark.pstats")

pr = cProfile.Profile()
pr.enable()

with benchmark_runner.assert_runtime(seconds=0.1, label=f"{request.node.name} - full block"):
# No transactions generator, the full node already cached it from the unfinished block
block_small = dataclasses.replace(block, transactions_generator=None)
res = await full_node_1.full_node.add_block(block_small)

log.warning(f"Res: {res}")

pr.create_stats()
pr.dump_stats("./full-block-benchmark.pstats")
12 changes: 6 additions & 6 deletions tests/core/mempool/test_mempool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2504,7 +2504,7 @@ def test_duplicate_large_integer_ladder(self, opcode, softfork_height, benchmark
def test_duplicate_large_integer(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner):
condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0x00")

with benchmark_runner.assert_runtime(seconds=1.1):
with benchmark_runner.assert_runtime(seconds=3):
npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)

assert npc_result.error == error_for_condition(opcode)
Expand All @@ -2521,7 +2521,7 @@ def test_duplicate_large_integer(self, opcode, softfork_height, benchmark_runner
def test_duplicate_large_integer_substr(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner):
condition = SINGLE_ARG_INT_SUBSTR_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0x00")

with benchmark_runner.assert_runtime(seconds=1.5):
with benchmark_runner.assert_runtime(seconds=2):
npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)

assert npc_result.error == error_for_condition(opcode)
Expand Down Expand Up @@ -2557,7 +2557,7 @@ def test_duplicate_large_integer_substr_tail(self, opcode, softfork_height, benc
def test_duplicate_large_integer_negative(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner):
condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0xff")

with benchmark_runner.assert_runtime(seconds=1):
with benchmark_runner.assert_runtime(seconds=2.75):
npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)

assert npc_result.error is None
Expand All @@ -2568,7 +2568,7 @@ def test_duplicate_reserve_fee(self, softfork_height, benchmark_runner: Benchmar
opcode = ConditionOpcode.RESERVE_FEE
condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0x00")

with benchmark_runner.assert_runtime(seconds=1):
with benchmark_runner.assert_runtime(seconds=1.5):
npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)

assert npc_result.error == error_for_condition(opcode)
Expand All @@ -2577,7 +2577,7 @@ def test_duplicate_reserve_fee_negative(self, softfork_height, benchmark_runner:
opcode = ConditionOpcode.RESERVE_FEE
condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=200000, val=100, filler="0xff")

with benchmark_runner.assert_runtime(seconds=0.8):
with benchmark_runner.assert_runtime(seconds=1.5):
npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)

# RESERVE_FEE conditions fail unconditionally if they have a negative
Expand All @@ -2596,7 +2596,7 @@ def test_duplicate_coin_announces(self, opcode, softfork_height, benchmark_runne
else:
condition = CREATE_ANNOUNCE_COND.format(opcode=opcode.value[0], num=5950000)

with benchmark_runner.assert_runtime(seconds=11):
with benchmark_runner.assert_runtime(seconds=14):
npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)

assert npc_result.error is None
Expand Down
4 changes: 2 additions & 2 deletions tests/core/test_cost_calculation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ async def test_tx_generator_speed(softfork_height, benchmark_runner: BenchmarkRu
generator_bytes = large_block_generator(LARGE_BLOCK_COIN_CONSUMED_COUNT)
program = SerializedProgram.from_bytes(generator_bytes)

with benchmark_runner.assert_runtime(seconds=0.5):
with benchmark_runner.assert_runtime(seconds=1.25):
generator = BlockGenerator(program, [], [])
npc_result = get_name_puzzle_conditions(
generator,
Expand Down Expand Up @@ -303,7 +303,7 @@ async def test_get_puzzle_and_solution_for_coin_performance(benchmark_runner: Be
# benchmark the function to pick out the puzzle and solution for a specific
# coin
generator = BlockGenerator(LARGE_BLOCK.transactions_generator, [], [])
with benchmark_runner.assert_runtime(seconds=7, label="get_puzzle_and_solution_for_coin"):
with benchmark_runner.assert_runtime(seconds=8.5, label="get_puzzle_and_solution_for_coin"):
for i in range(3):
for c in spends:
spend_info = get_puzzle_and_solution_for_coin(generator, c, 0, test_constants)
Expand Down
2 changes: 1 addition & 1 deletion tests/wallet/test_offer_parsing_performance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def enable_profiler(name: str) -> Iterator[None]:

def test_offer_parsing_performance(benchmark_runner: BenchmarkRunner) -> None:
offer_bytes = bytes.fromhex(test_offer)
with benchmark_runner.assert_runtime(seconds=2, label="Offer.from_bytes()"):
with benchmark_runner.assert_runtime(seconds=2.15, label="Offer.from_bytes()"):
with enable_profiler("offer-parsing"):
for _ in range(100):
o = Offer.from_bytes(offer_bytes)
Expand Down