add time out assert data collection and presentation (#16831)

* add tests.ether

* add new tests/ether.py

* go time out assert

* remove unused JunitPropertyName

* rename to `tests/process_junit.py`

* fixup

* hinting and pylint

* fixup

* after install

* handle skipped tests

* --only plot_sync/

* better relative paths

* better relative paths

* todos

* todos

* tidy

* back to full test run

* add markdown headings

* just make results an artifact

$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of 1024k, got 6067k.

* also output the top 50 to the step summary

* Update process_junit.py

* Update process_junit.py

.github/CODEOWNERS

@@ -2,3 +2,4 @@
 /.github/* @Chia-Network/actions-reviewers
 /PRETTY_GOOD_PRACTICES.md	@altendky @Chia-Network/required-reviewers
 /pylintrc	@altendky @Chia-Network/required-reviewers
+/tests/ether.py @altendky @Chia-Network/required-reviewers

.github/workflows/benchmarks.yml

@@ -124,4 +124,4 @@ jobs:
       - name: Add benchmark results to workflow summary
         if: always()
         run: |
-          python -m tests.process_benchmarks --xml junit-data/benchmarks.xml --markdown --link-prefix ${{ github.event.repository.html_url }}/blob/${{ github.sha }}/ --link-line-separator \#L >> "$GITHUB_STEP_SUMMARY"
+          python -m tests.process_junit --type benchmark --xml junit-data/benchmarks.xml --markdown --link-prefix ${{ github.event.repository.html_url }}/blob/${{ github.sha }}/ --link-line-separator \#L >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test.yml

@@ -142,14 +142,6 @@ jobs:
           path: junit-data/*
           if-no-files-found: error
 
-      - name: Publish JUnit results
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: junit-results
-          path: junit-results/*
-          if-no-files-found: error
-
       - name: Download Coverage
         uses: actions/download-artifact@v4
         with:
@@ -169,6 +161,20 @@ jobs:
 
       - uses: chia-network/actions/activate-venv@main
 
+      - name: Add time out assert results to workflow summary
+        if: always()
+        run: |
+          python -m tests.process_junit --limit 50 --type time_out_assert --xml junit-results/junit.xml --markdown --link-prefix ${{ github.event.repository.html_url }}/blob/${{ github.sha }}/ --link-line-separator \#L >> "$GITHUB_STEP_SUMMARY"
+          python -m tests.process_junit            --type time_out_assert --xml junit-results/junit.xml --markdown --link-prefix ${{ github.event.repository.html_url }}/blob/${{ github.sha }}/ --link-line-separator \#L >> junit-results/time_out_assert.md
+
+      - name: Publish JUnit results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: junit-results
+          path: junit-results/*
+          if-no-files-found: error
+
       - name: Coverage Processing
         run: |
           coverage combine --rcfile=.coveragerc --data-file=coverage-reports/.coverage coverage-data/

chia/util/misc.py

@@ -8,6 +8,7 @@
 import signal
 import sys
 from dataclasses import dataclass
+from inspect import getframeinfo, stack
 from pathlib import Path
 from types import FrameType
 from typing import (
@@ -19,10 +20,12 @@
     ContextManager,
     Dict,
     Generic,
+    Iterable,
     Iterator,
     List,
     Optional,
     Sequence,
+    Tuple,
     TypeVar,
     Union,
     final,
@@ -421,3 +424,17 @@ def available_logical_cores() -> int:
         return count
 
     return len(psutil.Process().cpu_affinity())
+
+
+def caller_file_and_line(distance: int = 1, relative_to: Iterable[Path] = ()) -> Tuple[str, int]:
+    caller = getframeinfo(stack()[distance + 1][0])
+
+    caller_path = Path(caller.filename)
+    options: List[str] = [caller_path.as_posix()]
+    for path in relative_to:
+        try:
+            options.append(caller_path.relative_to(path).as_posix())
+        except ValueError:
+            pass
+
+    return min(options, key=len), caller.lineno

tests/conftest.py

@@ -5,6 +5,7 @@
 import dataclasses
 import datetime
 import functools
+import json
 import math
 import multiprocessing
 import os
@@ -20,7 +21,9 @@
 
 # TODO: update after resolution in https://github.com/pytest-dev/pytest/issues/7469
 from _pytest.fixtures import SubRequest
+from pytest import MonkeyPatch
 
+import tests
 from chia.clvm.spend_sim import CostLogger
 from chia.consensus.constants import ConsensusConstants
 from chia.full_node.full_node import FullNode
@@ -66,10 +69,11 @@
 from chia.util.task_timing import start_task_instrumentation, stop_task_instrumentation
 from chia.wallet.wallet_node import WalletNode
 from chia.wallet.wallet_node_api import WalletNodeAPI
+from tests import ether
 from tests.core.data_layer.util import ChiaRoot
 from tests.core.node_height import node_height_at_least
 from tests.simulation.test_simulation import test_constants_modified
-from tests.util.misc import BenchmarkRunner, GcMode, RecordingWebServer, _AssertRuntime, measure_overhead
+from tests.util.misc import BenchmarkRunner, GcMode, RecordingWebServer, TestId, _AssertRuntime, measure_overhead
 from tests.util.setup_nodes import (
     OldSimulatorsAndWallets,
     SimulatorsAndWallets,
@@ -91,6 +95,20 @@
 from tests.util.setup_nodes import setup_farmer_multi_harvester
 
 
+@pytest.fixture(name="ether_setup", autouse=True)
+def ether_setup_fixture(request: SubRequest, record_property: Callable[[str, object], None]) -> Iterator[None]:
+    with MonkeyPatch.context() as monkeypatch_context:
+        monkeypatch_context.setattr(ether, "record_property", record_property)
+        monkeypatch_context.setattr(ether, "test_id", TestId.create(node=request.node))
+        yield
+
+
+@pytest.fixture(autouse=True)
+def ether_test_id_property_fixture(ether_setup: None, record_property: Callable[[str, object], None]) -> None:
+    assert ether.test_id is not None, "ether.test_id is None, did you forget to use the ether_setup fixture?"
+    record_property("test_id", json.dumps(ether.test_id.marshal(), ensure_ascii=True, sort_keys=True))
+
+
 def make_old_setup_simulators_and_wallets(new: SimulatorsAndWallets) -> OldSimulatorsAndWallets:
     return (
         [simulator.peer_api for simulator in new.simulators],
@@ -131,16 +149,12 @@ def benchmark_runner_overhead_fixture() -> float:
 
 @pytest.fixture(name="benchmark_runner")
 def benchmark_runner_fixture(
-    request: SubRequest,
     benchmark_runner_overhead: float,
-    record_property: Callable[[str, object], None],
     benchmark_repeat: int,
 ) -> BenchmarkRunner:
-    label = request.node.name
     return BenchmarkRunner(
-        label=label,
+        test_id=ether.test_id,
         overhead=benchmark_runner_overhead,
-        record_property=record_property,
     )
 
 
@@ -434,6 +448,13 @@ def pytest_addoption(parser: pytest.Parser):
         type=int,
         help=f"The number of times to run each benchmark, default {default_repeats}.",
     )
+    group.addoption(
+        "--time-out-assert-repeats",
+        action="store",
+        default=default_repeats,
+        type=int,
+        help=f"The number of times to run each test with time out asserts, default {default_repeats}.",
+    )
 
 
 def pytest_configure(config):
@@ -459,6 +480,22 @@ def benchmark_repeat_fixture() -> int:
 
     globals()[benchmark_repeat_fixture.__name__] = benchmark_repeat_fixture
 
+    time_out_assert_repeats = config.getoption("--time-out-assert-repeats")
+    if time_out_assert_repeats != 1:
+
+        @pytest.fixture(
+            name="time_out_assert_repeat",
+            autouse=True,
+            params=[
+                pytest.param(repeat, id=f"time_out_assert_repeat{repeat:03d}")
+                for repeat in range(time_out_assert_repeats)
+            ],
+        )
+        def time_out_assert_repeat_fixture(request: SubRequest) -> int:
+            return request.param
+
+        globals()[time_out_assert_repeat_fixture.__name__] = time_out_assert_repeat_fixture
+
 
 def pytest_collection_modifyitems(session, config: pytest.Config, items: List[pytest.Function]):
     # https://github.com/pytest-dev/pytest/issues/3730#issuecomment-567142496

tests/ether.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable, Optional
+
+if TYPE_CHECKING:
+    from tests.util.misc import TestId
+
+# NOTE: Do not just put any useful thing here.  This is specifically for making
+#       fixture values globally available during tests.  In _most_ cases fixtures
+#       should be directly requested using normal mechanisms.  Very little should
+#       be put here.
+
+# NOTE: When using this module do not import the attributes directly.  Rather, import
+#       something like `from tests import ether`.  Importing attributes directly will
+#       result in you likely getting the default `None` values since they are not
+#       populated until tests are running.
+
+record_property: Optional[Callable[[str, object], None]] = None
+test_id: Optional[TestId] = None