From fe66ff9cd90bf365a5ebd8c83654d2ab24ab3bec Mon Sep 17 00:00:00 2001 From: Florin Chirica Date: Wed, 18 Oct 2023 02:59:28 +0300 Subject: [PATCH 1/4] Fix benchmark flakiness --- .github/workflows/benchmarks.yml | 2 +- tests/core/full_node/test_performance.py | 2 +- tests/core/mempool/test_mempool.py | 12 ++++++------ tests/core/mempool/test_mempool_performance.py | 4 ++-- tests/core/test_cost_calculation.py | 4 ++-- tests/wallet/test_offer_parsing_performance.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 46644a01554f..fee912b5b270 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -28,7 +28,7 @@ jobs: runs-on: benchmark container: image: ubuntu:22.04 - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: diff --git a/tests/core/full_node/test_performance.py b/tests/core/full_node/test_performance.py index 76dbb8d5909f..d97201e5231d 100644 --- a/tests/core/full_node/test_performance.py +++ b/tests/core/full_node/test_performance.py @@ -119,7 +119,7 @@ async def test_full_block_performance( pr = cProfile.Profile() pr.enable() - with benchmark_runner.assert_runtime(seconds=0.001, label=f"{request.node.name} - mempool"): + with benchmark_runner.assert_runtime(seconds=0.0055, label=f"{request.node.name} - mempool"): num_tx: int = 0 for spend_bundle, spend_bundle_id in zip(spend_bundles, spend_bundle_ids): num_tx += 1 diff --git a/tests/core/mempool/test_mempool.py b/tests/core/mempool/test_mempool.py index 167171be3e4d..589bdde3ca3e 100644 --- a/tests/core/mempool/test_mempool.py +++ b/tests/core/mempool/test_mempool.py @@ -2504,7 +2504,7 @@ def test_duplicate_large_integer_ladder(self, opcode, softfork_height, benchmark def test_duplicate_large_integer(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner): condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0x00") - with benchmark_runner.assert_runtime(seconds=1.1): + with benchmark_runner.assert_runtime(seconds=3): npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) assert npc_result.error == error_for_condition(opcode) @@ -2521,7 +2521,7 @@ def test_duplicate_large_integer(self, opcode, softfork_height, benchmark_runner def test_duplicate_large_integer_substr(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner): condition = SINGLE_ARG_INT_SUBSTR_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0x00") - with benchmark_runner.assert_runtime(seconds=1.5): + with benchmark_runner.assert_runtime(seconds=2): npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) assert npc_result.error == error_for_condition(opcode) @@ -2557,7 +2557,7 @@ def test_duplicate_large_integer_substr_tail(self, opcode, softfork_height, benc def test_duplicate_large_integer_negative(self, opcode, softfork_height, benchmark_runner: BenchmarkRunner): condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0xff") - with benchmark_runner.assert_runtime(seconds=1): + with benchmark_runner.assert_runtime(seconds=2.75): npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) assert npc_result.error is None @@ -2568,7 +2568,7 @@ def test_duplicate_reserve_fee(self, softfork_height, benchmark_runner: Benchmar opcode = ConditionOpcode.RESERVE_FEE condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=280000, val=100, filler="0x00") - with benchmark_runner.assert_runtime(seconds=1): + with benchmark_runner.assert_runtime(seconds=1.5): npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) assert npc_result.error == error_for_condition(opcode) @@ -2577,7 +2577,7 @@ def test_duplicate_reserve_fee_negative(self, softfork_height, benchmark_runner: opcode = ConditionOpcode.RESERVE_FEE condition = SINGLE_ARG_INT_COND.format(opcode=opcode.value[0], num=200000, val=100, filler="0xff") - with benchmark_runner.assert_runtime(seconds=0.8): + with benchmark_runner.assert_runtime(seconds=1.5): npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) # RESERVE_FEE conditions fail unconditionally if they have a negative @@ -2596,7 +2596,7 @@ def test_duplicate_coin_announces(self, opcode, softfork_height, benchmark_runne else: condition = CREATE_ANNOUNCE_COND.format(opcode=opcode.value[0], num=5950000) - with benchmark_runner.assert_runtime(seconds=11): + with benchmark_runner.assert_runtime(seconds=14): npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) assert npc_result.error is None diff --git a/tests/core/mempool/test_mempool_performance.py b/tests/core/mempool/test_mempool_performance.py index 24ff71317cb7..688448938b62 100644 --- a/tests/core/mempool/test_mempool_performance.py +++ b/tests/core/mempool/test_mempool_performance.py @@ -76,9 +76,9 @@ async def test_mempool_update_performance( for idx, block in enumerate(blocks): if idx >= len(blocks) - 3: - duration = 0.15 + duration = 0.2 else: - duration = 0.001 + duration = 0.01 with benchmark_runner.assert_runtime(seconds=duration): await full_node_api_1.full_node.add_block(block) diff --git a/tests/core/test_cost_calculation.py b/tests/core/test_cost_calculation.py index 2cae1bfdadd7..5442421efaeb 100644 --- a/tests/core/test_cost_calculation.py +++ b/tests/core/test_cost_calculation.py @@ -207,7 +207,7 @@ async def test_tx_generator_speed(softfork_height, benchmark_runner: BenchmarkRu generator_bytes = large_block_generator(LARGE_BLOCK_COIN_CONSUMED_COUNT) program = SerializedProgram.from_bytes(generator_bytes) - with benchmark_runner.assert_runtime(seconds=0.5): + with benchmark_runner.assert_runtime(seconds=1.25): generator = BlockGenerator(program, [], []) npc_result = get_name_puzzle_conditions( generator, @@ -303,7 +303,7 @@ async def test_get_puzzle_and_solution_for_coin_performance(benchmark_runner: Be # benchmark the function to pick out the puzzle and solution for a specific # coin generator = BlockGenerator(LARGE_BLOCK.transactions_generator, [], []) - with benchmark_runner.assert_runtime(seconds=7, label="get_puzzle_and_solution_for_coin"): + with benchmark_runner.assert_runtime(seconds=8.5, label="get_puzzle_and_solution_for_coin"): for i in range(3): for c in spends: spend_info = get_puzzle_and_solution_for_coin(generator, c, 0, test_constants) diff --git a/tests/wallet/test_offer_parsing_performance.py b/tests/wallet/test_offer_parsing_performance.py index 38031886f39d..e8755e17fb90 100644 --- a/tests/wallet/test_offer_parsing_performance.py +++ b/tests/wallet/test_offer_parsing_performance.py @@ -28,7 +28,7 @@ def enable_profiler(name: str) -> Iterator[None]: def test_offer_parsing_performance(benchmark_runner: BenchmarkRunner) -> None: offer_bytes = bytes.fromhex(test_offer) - with benchmark_runner.assert_runtime(seconds=2, label="Offer.from_bytes()"): + with benchmark_runner.assert_runtime(seconds==2.15, label="Offer.from_bytes()"): with enable_profiler("offer-parsing"): for _ in range(100): o = Offer.from_bytes(offer_bytes) From 8336c4ce9434c0f7f377fb3037fcee2e16c4d295 Mon Sep 17 00:00:00 2001 From: Florin Chirica Date: Wed, 18 Oct 2023 03:02:22 +0300 Subject: [PATCH 2/4] Update test_offer_parsing_performance.py --- tests/wallet/test_offer_parsing_performance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wallet/test_offer_parsing_performance.py b/tests/wallet/test_offer_parsing_performance.py index e8755e17fb90..d3bf769accdf 100644 --- a/tests/wallet/test_offer_parsing_performance.py +++ b/tests/wallet/test_offer_parsing_performance.py @@ -28,7 +28,7 @@ def enable_profiler(name: str) -> Iterator[None]: def test_offer_parsing_performance(benchmark_runner: BenchmarkRunner) -> None: offer_bytes = bytes.fromhex(test_offer) - with benchmark_runner.assert_runtime(seconds==2.15, label="Offer.from_bytes()"): + with benchmark_runner.assert_runtime(seconds=2.15, label="Offer.from_bytes()"): with enable_profiler("offer-parsing"): for _ in range(100): o = Offer.from_bytes(offer_bytes) From 708e24eb450063df0810726154d1d4b005588b06 Mon Sep 17 00:00:00 2001 From: Florin Chirica Date: Fri, 20 Oct 2023 15:52:26 +0300 Subject: [PATCH 3/4] Remove profilers. --- .github/workflows/benchmarks.yml | 2 +- tests/core/full_node/test_performance.py | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index fee912b5b270..46644a01554f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -28,7 +28,7 @@ jobs: runs-on: benchmark container: image: ubuntu:22.04 - timeout-minutes: 60 + timeout-minutes: 30 strategy: fail-fast: false matrix: diff --git a/tests/core/full_node/test_performance.py b/tests/core/full_node/test_performance.py index d97201e5231d..eb0e50ed7c4c 100644 --- a/tests/core/full_node/test_performance.py +++ b/tests/core/full_node/test_performance.py @@ -1,7 +1,6 @@ # flake8: noqa: F811, F401 from __future__ import annotations -import cProfile import dataclasses import logging import random @@ -116,9 +115,6 @@ async def test_full_block_performance( spend_bundles.append(spend_bundle) spend_bundle_ids.append(spend_bundle.get_hash()) - pr = cProfile.Profile() - pr.enable() - with benchmark_runner.assert_runtime(seconds=0.0055, label=f"{request.node.name} - mempool"): num_tx: int = 0 for spend_bundle, spend_bundle_id in zip(spend_bundles, spend_bundle_ids): @@ -173,9 +169,6 @@ async def test_full_block_performance( [], ) - pr = cProfile.Profile() - pr.enable() - with benchmark_runner.assert_runtime(seconds=0.1, label=f"{request.node.name} - unfinished"): res = await full_node_1.respond_unfinished_block(fnp.RespondUnfinishedBlock(unfinished), fake_peer) @@ -184,9 +177,6 @@ async def test_full_block_performance( pr.create_stats() pr.dump_stats("./unfinished-benchmark.pstats") - pr = cProfile.Profile() - pr.enable() - with benchmark_runner.assert_runtime(seconds=0.1, label=f"{request.node.name} - full block"): # No transactions generator, the full node already cached it from the unfinished block block_small = dataclasses.replace(block, transactions_generator=None) From 755190d7f69788b8082ccf8096a10eb9de4139f4 Mon Sep 17 00:00:00 2001 From: Florin Chirica Date: Fri, 20 Oct 2023 15:57:42 +0300 Subject: [PATCH 4/4] Further remove profilers. --- tests/core/full_node/test_performance.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/core/full_node/test_performance.py b/tests/core/full_node/test_performance.py index eb0e50ed7c4c..8522883e25bf 100644 --- a/tests/core/full_node/test_performance.py +++ b/tests/core/full_node/test_performance.py @@ -130,8 +130,6 @@ async def test_full_block_performance( break log.warning(f"Num Tx: {num_tx}") - pr.create_stats() - pr.dump_stats("./mempool-benchmark.pstats") # Create an unfinished block peak = full_node_1.full_node.blockchain.get_peak() @@ -174,15 +172,9 @@ async def test_full_block_performance( log.warning(f"Res: {res}") - pr.create_stats() - pr.dump_stats("./unfinished-benchmark.pstats") - with benchmark_runner.assert_runtime(seconds=0.1, label=f"{request.node.name} - full block"): # No transactions generator, the full node already cached it from the unfinished block block_small = dataclasses.replace(block, transactions_generator=None) res = await full_node_1.full_node.add_block(block_small) log.warning(f"Res: {res}") - - pr.create_stats() - pr.dump_stats("./full-block-benchmark.pstats")