Fix benchmark flakiness (#16649)

Chia-Network · Oct 24, 2023 · ca59c50 · ca59c50
1 parent ef98949
commit ca59c50
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 27 deletions.
diff --git a/tests/core/full_node/test_performance.py b/tests/core/full_node/test_performance.py
@@ -1,7 +1,6 @@
 # flake8: noqa: F811, F401
 from __future__ import annotations
 
-import cProfile
 import dataclasses
 import logging
 import random
@@ -116,10 +115,7 @@ async def test_full_block_performance(
             spend_bundles.append(spend_bundle)
             spend_bundle_ids.append(spend_bundle.get_hash())
 
-        pr = cProfile.Profile()
-        pr.enable()
-
-        with benchmark_runner.assert_runtime(seconds=0.001, label=f"{request.node.name} - mempool"):
+        with benchmark_runner.assert_runtime(seconds=0.0055, label=f"{request.node.name} - mempool"):
             num_tx: int = 0
             for spend_bundle, spend_bundle_id in zip(spend_bundles, spend_bundle_ids):
                 num_tx += 1
@@ -134,8 +130,6 @@ async def test_full_block_performance(
                     break
 
         log.warning(f"Num Tx: {num_tx}")
-        pr.create_stats()
-        pr.dump_stats("./mempool-benchmark.pstats")
 
         # Create an unfinished block
         peak = full_node_1.full_node.blockchain.get_peak()
@@ -173,26 +167,14 @@ async def test_full_block_performance(
             [],
         )
 
-        pr = cProfile.Profile()
-        pr.enable()
-
         with benchmark_runner.assert_runtime(seconds=0.1, label=f"{request.node.name} - unfinished"):
             res = await full_node_1.respond_unfinished_block(fnp.RespondUnfinishedBlock(unfinished), fake_peer)
 
         log.warning(f"Res: {res}")
 
-        pr.create_stats()
-        pr.dump_stats("./unfinished-benchmark.pstats")
-
-        pr = cProfile.Profile()
-        pr.enable()
-
         with benchmark_runner.assert_runtime(seconds=0.1, label=f"{request.node.name} - full block"):
             # No transactions generator, the full node already cached it from the unfinished block
             block_small = dataclasses.replace(block, transactions_generator=None)
             res = await full_node_1.full_node.add_block(block_small)
 
         log.warning(f"Res: {res}")
-
-        pr.create_stats()
-        pr.dump_stats("./full-block-benchmark.pstats")
diff --git a/tests/core/mempool/test_mempool.py b/tests/core/mempool/test_mempool.py
@@ -2504,7 +2504,7 @@ def test_duplicate_large_integer_ladder(self, opcode, softfork_height, benchmark
     def test_duplicate_large_integer(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner):
         condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0x00")
 
-        with benchmark_runner.assert_runtime(seconds=1.1):
+        with benchmark_runner.assert_runtime(seconds=3):
             npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)
 
         assert npc_result.error == error_for_condition(opcode)
@@ -2521,7 +2521,7 @@ def test_duplicate_large_integer(self, opcode, softfork_height, benchmark_runner
     def test_duplicate_large_integer_substr(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner):
         condition = SINGLE_ARG_INT_SUBSTR_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0x00")
 
-        with benchmark_runner.assert_runtime(seconds=1.5):
+        with benchmark_runner.assert_runtime(seconds=2):
             npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)
 
         assert npc_result.error == error_for_condition(opcode)
@@ -2557,7 +2557,7 @@ def test_duplicate_large_integer_substr_tail(self, opcode, softfork_height, benc
     def test_duplicate_large_integer_negative(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner):
         condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0xff")
 
-        with benchmark_runner.assert_runtime(seconds=1):
+        with benchmark_runner.assert_runtime(seconds=2.75):
             npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)
 
         assert npc_result.error is None
@@ -2577,7 +2577,7 @@ def test_duplicate_reserve_fee_negative(self, softfork_height, benchmark_runner:
         opcode = ConditionOpcode.RESERVE_FEE
         condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=200000, val=100, filler="0xff")
 
-        with benchmark_runner.assert_runtime(seconds=0.8):
+        with benchmark_runner.assert_runtime(seconds=1.5):
             npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)
 
         # RESERVE_FEE conditions fail unconditionally if they have a negative
@@ -2596,7 +2596,7 @@ def test_duplicate_coin_announces(self, opcode, softfork_height, benchmark_runne
         else:
             condition = CREATE_ANNOUNCE_COND.format(opcode=opcode.value[0], num=5950000)
 
-        with benchmark_runner.assert_runtime(seconds=11):
+        with benchmark_runner.assert_runtime(seconds=14):
             npc_result = generator_condition_tester(condition, quote=False, height=softfork_height)
 
         assert npc_result.error is None

diff --git a/tests/core/test_cost_calculation.py b/tests/core/test_cost_calculation.py
@@ -207,7 +207,7 @@ async def test_tx_generator_speed(softfork_height, benchmark_runner: BenchmarkRu
     generator_bytes = large_block_generator(LARGE_BLOCK_COIN_CONSUMED_COUNT)
     program = SerializedProgram.from_bytes(generator_bytes)
 
-    with benchmark_runner.assert_runtime(seconds=0.5):
+    with benchmark_runner.assert_runtime(seconds=1.25):
         generator = BlockGenerator(program, [], [])
         npc_result = get_name_puzzle_conditions(
             generator,
@@ -303,7 +303,7 @@ async def test_get_puzzle_and_solution_for_coin_performance(benchmark_runner: Be
     # benchmark the function to pick out the puzzle and solution for a specific
     # coin
     generator = BlockGenerator(LARGE_BLOCK.transactions_generator, [], [])
-    with benchmark_runner.assert_runtime(seconds=7, label="get_puzzle_and_solution_for_coin"):
+    with benchmark_runner.assert_runtime(seconds=8.5, label="get_puzzle_and_solution_for_coin"):
         for i in range(3):
             for c in spends:
                 spend_info = get_puzzle_and_solution_for_coin(generator, c, 0, test_constants)

diff --git a/tests/wallet/test_offer_parsing_performance.py b/tests/wallet/test_offer_parsing_performance.py
@@ -28,7 +28,7 @@ def enable_profiler(name: str) -> Iterator[None]:
 
 def test_offer_parsing_performance(benchmark_runner: BenchmarkRunner) -> None:
     offer_bytes = bytes.fromhex(test_offer)
-    with benchmark_runner.assert_runtime(seconds=2, label="Offer.from_bytes()"):
+    with benchmark_runner.assert_runtime(seconds=2.15, label="Offer.from_bytes()"):
         with enable_profiler("offer-parsing"):
             for _ in range(100):
                 o = Offer.from_bytes(offer_bytes)