Spec test coverage report hack (#13718)

* Spec test report hack

* no export

* fix shell complaint

* shell fix?

* shell again?

* chmod +x ./hack/spectest-report.sh

* Review + improvements

* Remove unwanted change

* Add exclusion list

* Fix path + add eip6110 to exclusion

* Fix bazel path nonsense

* Add extra detail about specific test

* Cleanup exclusion list

* Add fail conditions

* Add mkdir

* Shorten filename + mkdir only if new

* Fix names

* Add to exclusion list

* Add report to .gitignore

* Back to stupid names

* Add Bazel flags option

---------

Co-authored-by: Preston Van Loon <pvanloon@offchainlabs.com>

.gitignore

@@ -41,3 +41,6 @@ jwt.hex
 
 # manual testing
 tmp
+
+# spectest coverage reports
+report.txt

bazel.sh

@@ -2,7 +2,7 @@
 
 # This script serves as a wrapper around bazel to limit the scope of environment variables that
 # may change the action output. Using this script should result in a higher cache hit ratio for
-# cached actions with a more heremtic build.
+# cached actions with a more hermetic build.
 
 env -i \
  PATH=/usr/bin:/bin \

hack/spectest-report.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -xe
+
+# Constants
+PROJECT_ROOT=$(pwd)
+PRYSM_DIR="${PROJECT_ROOT%/hack}/testing/spectest"
+EXCLUSION_LIST="$PRYSM_DIR/exclusions.txt"
+BAZEL_DIR="/tmp/spectest_report"
+SPEC_REPO="git@github.com:ethereum/consensus-spec-tests.git"
+SPEC_DIR="/tmp/consensus-spec"
+
+# Create directory if it doesn't already exist
+mkdir -p "$BAZEL_DIR"
+
+# Add any passed flags to BAZEL_FLAGS
+BAZEL_FLAGS=""
+for flag in "$@"
+do
+    BAZEL_FLAGS="$BAZEL_FLAGS $flag"
+done
+
+# Run spectests
+bazel test //testing/spectest/... --test_env=SPEC_TEST_REPORT_OUTPUT_DIR="$BAZEL_DIR" $BAZEL_FLAGS
+
+# Ensure the SPEC_DIR exists and is a git repository
+if [ -d "$SPEC_DIR/.git" ]; then
+    echo "Repository already exists. Pulling latest changes."
+    (cd "$SPEC_DIR" && git pull) || exit 1
+else
+    echo "Cloning the GitHub repository."
+    git clone "$SPEC_REPO" "$SPEC_DIR" || exit 1
+fi
+
+# Finding all *_tests.txt files in BAZEL_DIR and concatenating them into tests.txt
+find "$BAZEL_DIR" -type f -name '*_tests.txt' -exec cat {} + > "$PRYSM_DIR/tests.txt"
+
+# Generating spec.txt
+(cd "$SPEC_DIR" && find tests -maxdepth 4 -mindepth 4 -type d > "$PRYSM_DIR/spec.txt") || exit 1
+
+# Comparing spec.txt with tests.txt and generating report.txt
+while IFS= read -r line; do
+   if grep -Fxq "$line" "$EXCLUSION_LIST"; then
+      # If it's excluded and we have a test for it flag as an error
+      if grep -q "$line" "$PRYSM_DIR/tests.txt"; then
+          echo "Error: Excluded item found in tests.txt: $line"
+          exit 1 # Exit with an error status
+      else
+          echo "Skipping excluded item: $line"
+      fi
+          continue
+    fi
+    if grep -q "$line" "$PRYSM_DIR/tests.txt"; then
+        echo "found $line"
+    else
+        echo "missing $line"
+    fi
+done < "$PRYSM_DIR/spec.txt" > "$PRYSM_DIR/report.txt"
+
+# Formatting report.txt
+{
+    echo "Prysm Spectest Report"
+    echo ""
+    echo "Tests Missing"
+    grep '^missing' "$PRYSM_DIR/report.txt"
+    echo ""
+    echo "Tests Found"
+    grep '^found' "$PRYSM_DIR/report.txt"
+} > "$PRYSM_DIR/report_temp.txt" && mv "$PRYSM_DIR/report_temp.txt" "$PRYSM_DIR/report.txt"
+
+# Check for the word "missing" in the report and exit with an error if present
+if grep -q '^missing' "$PRYSM_DIR/report.txt"; then
+    echo "Error: 'missing' tests found in report: $PRYSM_DIR/report.txt"
+    exit 1
+fi
+
+# Clean up
+rm -f "$PRYSM_DIR/tests.txt" "$PRYSM_DIR/spec.txt"

testing/spectest/exclusions.txt

@@ -0,0 +1,296 @@
+# LightClient
+tests/mainnet/altair/light_client/single_merkle_proof
+tests/mainnet/bellatrix/light_client/single_merkle_proof
+tests/mainnet/capella/light_client/single_merkle_proof
+tests/mainnet/deneb/light_client/single_merkle_proof
+tests/minimal/altair/light_client/single_merkle_proof
+tests/minimal/altair/light_client/sync
+tests/minimal/altair/light_client/update_ranking
+tests/minimal/bellatrix/light_client/single_merkle_proof
+tests/minimal/bellatrix/light_client/sync
+tests/minimal/bellatrix/light_client/update_ranking
+tests/minimal/capella/light_client/single_merkle_proof
+tests/minimal/capella/light_client/sync
+tests/minimal/capella/light_client/update_ranking
+tests/minimal/deneb/light_client/single_merkle_proof
+tests/minimal/deneb/light_client/sync
+tests/minimal/deneb/light_client/update_ranking
+
+# SSZ Generic
+tests/general/phase0/ssz_generic/basic_vector
+tests/general/phase0/ssz_generic/bitlist
+tests/general/phase0/ssz_generic/bitvector
+tests/general/phase0/ssz_generic/boolean
+tests/general/phase0/ssz_generic/containers
+tests/general/phase0/ssz_generic/uints
+
+# EIP6110
+tests/mainnet/eip6110/epoch_processing/effective_balance_updates
+tests/mainnet/eip6110/epoch_processing/eth1_data_reset
+tests/mainnet/eip6110/epoch_processing/historical_summaries_update
+tests/mainnet/eip6110/epoch_processing/inactivity_updates
+tests/mainnet/eip6110/epoch_processing/justification_and_finalization
+tests/mainnet/eip6110/epoch_processing/participation_flag_updates
+tests/mainnet/eip6110/epoch_processing/randao_mixes_reset
+tests/mainnet/eip6110/epoch_processing/registry_updates
+tests/mainnet/eip6110/epoch_processing/rewards_and_penalties
+tests/mainnet/eip6110/epoch_processing/slashings
+tests/mainnet/eip6110/epoch_processing/slashings_reset
+tests/mainnet/eip6110/finality/finality
+tests/mainnet/eip6110/fork_choice/ex_ante
+tests/mainnet/eip6110/fork_choice/get_head
+tests/mainnet/eip6110/fork_choice/get_proposer_head
+tests/mainnet/eip6110/fork_choice/on_block
+tests/mainnet/eip6110/fork_choice/should_override_forkchoice_update
+tests/mainnet/eip6110/operations/attestation
+tests/mainnet/eip6110/operations/attester_slashing
+tests/mainnet/eip6110/operations/block_header
+tests/mainnet/eip6110/operations/bls_to_execution_change
+tests/mainnet/eip6110/operations/deposit
+tests/mainnet/eip6110/operations/deposit_receipt
+tests/mainnet/eip6110/operations/execution_payload
+tests/mainnet/eip6110/operations/proposer_slashing
+tests/mainnet/eip6110/operations/sync_aggregate
+tests/mainnet/eip6110/operations/voluntary_exit
+tests/mainnet/eip6110/operations/withdrawals
+tests/mainnet/eip6110/rewards/basic
+tests/mainnet/eip6110/rewards/leak
+tests/mainnet/eip6110/rewards/random
+tests/mainnet/eip6110/sanity/blocks
+tests/mainnet/eip6110/sanity/slots
+tests/mainnet/eip6110/ssz_static/AggregateAndProof
+tests/mainnet/eip6110/ssz_static/Attestation
+tests/mainnet/eip6110/ssz_static/AttestationData
+tests/mainnet/eip6110/ssz_static/AttesterSlashing
+tests/mainnet/eip6110/ssz_static/BLSToExecutionChange
+tests/mainnet/eip6110/ssz_static/BeaconBlock
+tests/mainnet/eip6110/ssz_static/BeaconBlockBody
+tests/mainnet/eip6110/ssz_static/BeaconBlockHeader
+tests/mainnet/eip6110/ssz_static/BeaconState
+tests/mainnet/eip6110/ssz_static/BlobIdentifier
+tests/mainnet/eip6110/ssz_static/BlobSidecar
+tests/mainnet/eip6110/ssz_static/Checkpoint
+tests/mainnet/eip6110/ssz_static/ContributionAndProof
+tests/mainnet/eip6110/ssz_static/Deposit
+tests/mainnet/eip6110/ssz_static/DepositData
+tests/mainnet/eip6110/ssz_static/DepositMessage
+tests/mainnet/eip6110/ssz_static/DepositReceipt
+tests/mainnet/eip6110/ssz_static/Eth1Block
+tests/mainnet/eip6110/ssz_static/Eth1Data
+tests/mainnet/eip6110/ssz_static/ExecutionPayload
+tests/mainnet/eip6110/ssz_static/ExecutionPayloadHeader
+tests/mainnet/eip6110/ssz_static/Fork
+tests/mainnet/eip6110/ssz_static/ForkData
+tests/mainnet/eip6110/ssz_static/HistoricalBatch
+tests/mainnet/eip6110/ssz_static/HistoricalSummary
+tests/mainnet/eip6110/ssz_static/IndexedAttestation
+tests/mainnet/eip6110/ssz_static/LightClientBootstrap
+tests/mainnet/eip6110/ssz_static/LightClientFinalityUpdate
+tests/mainnet/eip6110/ssz_static/LightClientHeader
+tests/mainnet/eip6110/ssz_static/LightClientOptimisticUpdate
+tests/mainnet/eip6110/ssz_static/LightClientUpdate
+tests/mainnet/eip6110/ssz_static/PendingAttestation
+tests/mainnet/eip6110/ssz_static/PowBlock
+tests/mainnet/eip6110/ssz_static/ProposerSlashing
+tests/mainnet/eip6110/ssz_static/SignedAggregateAndProof
+tests/mainnet/eip6110/ssz_static/SignedBLSToExecutionChange
+tests/mainnet/eip6110/ssz_static/SignedBeaconBlock
+tests/mainnet/eip6110/ssz_static/SignedBeaconBlockHeader
+tests/mainnet/eip6110/ssz_static/SignedContributionAndProof
+tests/mainnet/eip6110/ssz_static/SignedVoluntaryExit
+tests/mainnet/eip6110/ssz_static/SigningData
+tests/mainnet/eip6110/ssz_static/SyncAggregate
+tests/mainnet/eip6110/ssz_static/SyncAggregatorSelectionData
+tests/mainnet/eip6110/ssz_static/SyncCommittee
+tests/mainnet/eip6110/ssz_static/SyncCommitteeContribution
+tests/mainnet/eip6110/ssz_static/SyncCommitteeMessage
+tests/mainnet/eip6110/ssz_static/Validator
+tests/mainnet/eip6110/ssz_static/VoluntaryExit
+tests/mainnet/eip6110/ssz_static/Withdrawal
+tests/mainnet/eip6110/sync/optimistic
+tests/mainnet/eip6110/transition/core
+tests/minimal/eip6110/epoch_processing/effective_balance_updates
+tests/minimal/eip6110/epoch_processing/eth1_data_reset
+tests/minimal/eip6110/epoch_processing/historical_summaries_update
+tests/minimal/eip6110/epoch_processing/inactivity_updates
+tests/minimal/eip6110/epoch_processing/justification_and_finalization
+tests/minimal/eip6110/epoch_processing/participation_flag_updates
+tests/minimal/eip6110/epoch_processing/randao_mixes_reset
+tests/minimal/eip6110/epoch_processing/registry_updates
+tests/minimal/eip6110/epoch_processing/rewards_and_penalties
+tests/minimal/eip6110/epoch_processing/slashings
+tests/minimal/eip6110/epoch_processing/slashings_reset
+tests/minimal/eip6110/epoch_processing/sync_committee_updates
+tests/minimal/eip6110/finality/finality
+tests/minimal/eip6110/fork_choice/ex_ante
+tests/minimal/eip6110/fork_choice/get_head
+tests/minimal/eip6110/fork_choice/get_proposer_head
+tests/minimal/eip6110/fork_choice/on_block
+tests/minimal/eip6110/fork_choice/reorg
+tests/minimal/eip6110/fork_choice/should_override_forkchoice_update
+tests/minimal/eip6110/fork_choice/withholding
+tests/minimal/eip6110/genesis/initialization
+tests/minimal/eip6110/genesis/validity
+tests/minimal/eip6110/operations/attestation
+tests/minimal/eip6110/operations/attester_slashing
+tests/minimal/eip6110/operations/block_header
+tests/minimal/eip6110/operations/bls_to_execution_change
+tests/minimal/eip6110/operations/deposit
+tests/minimal/eip6110/operations/deposit_receipt
+tests/minimal/eip6110/operations/execution_payload
+tests/minimal/eip6110/operations/proposer_slashing
+tests/minimal/eip6110/operations/sync_aggregate
+tests/minimal/eip6110/operations/voluntary_exit
+tests/minimal/eip6110/operations/withdrawals
+tests/minimal/eip6110/rewards/basic
+tests/minimal/eip6110/rewards/leak
+tests/minimal/eip6110/rewards/random
+tests/minimal/eip6110/sanity/blocks
+tests/minimal/eip6110/sanity/slots
+tests/minimal/eip6110/ssz_static/AggregateAndProof
+tests/minimal/eip6110/ssz_static/Attestation
+tests/minimal/eip6110/ssz_static/AttestationData
+tests/minimal/eip6110/ssz_static/AttesterSlashing
+tests/minimal/eip6110/ssz_static/BLSToExecutionChange
+tests/minimal/eip6110/ssz_static/BeaconBlock
+tests/minimal/eip6110/ssz_static/BeaconBlockBody
+tests/minimal/eip6110/ssz_static/BeaconBlockHeader
+tests/minimal/eip6110/ssz_static/BeaconState
+tests/minimal/eip6110/ssz_static/BlobIdentifier
+tests/minimal/eip6110/ssz_static/BlobSidecar
+tests/minimal/eip6110/ssz_static/Checkpoint
+tests/minimal/eip6110/ssz_static/ContributionAndProof
+tests/minimal/eip6110/ssz_static/Deposit
+tests/minimal/eip6110/ssz_static/DepositData
+tests/minimal/eip6110/ssz_static/DepositMessage
+tests/minimal/eip6110/ssz_static/DepositReceipt
+tests/minimal/eip6110/ssz_static/Eth1Block
+tests/minimal/eip6110/ssz_static/Eth1Data
+tests/minimal/eip6110/ssz_static/ExecutionPayload
+tests/minimal/eip6110/ssz_static/ExecutionPayloadHeader
+tests/minimal/eip6110/ssz_static/Fork
+tests/minimal/eip6110/ssz_static/ForkData
+tests/minimal/eip6110/ssz_static/HistoricalBatch
+tests/minimal/eip6110/ssz_static/HistoricalSummary
+tests/minimal/eip6110/ssz_static/IndexedAttestation
+tests/minimal/eip6110/ssz_static/LightClientBootstrap
+tests/minimal/eip6110/ssz_static/LightClientFinalityUpdate
+tests/minimal/eip6110/ssz_static/LightClientHeader
+tests/minimal/eip6110/ssz_static/LightClientOptimisticUpdate
+tests/minimal/eip6110/ssz_static/LightClientUpdate
+tests/minimal/eip6110/ssz_static/PendingAttestation
+tests/minimal/eip6110/ssz_static/PowBlock
+tests/minimal/eip6110/ssz_static/ProposerSlashing
+tests/minimal/eip6110/ssz_static/SignedAggregateAndProof
+tests/minimal/eip6110/ssz_static/SignedBLSToExecutionChange
+tests/minimal/eip6110/ssz_static/SignedBeaconBlock
+tests/minimal/eip6110/ssz_static/SignedBeaconBlockHeader
+tests/minimal/eip6110/ssz_static/SignedContributionAndProof
+tests/minimal/eip6110/ssz_static/SignedVoluntaryExit
+tests/minimal/eip6110/ssz_static/SigningData
+tests/minimal/eip6110/ssz_static/SyncAggregate
+tests/minimal/eip6110/ssz_static/SyncAggregatorSelectionData
+tests/minimal/eip6110/ssz_static/SyncCommittee
+tests/minimal/eip6110/ssz_static/SyncCommitteeContribution
+tests/minimal/eip6110/ssz_static/SyncCommitteeMessage
+tests/minimal/eip6110/ssz_static/Validator
+tests/minimal/eip6110/ssz_static/VoluntaryExit
+tests/minimal/eip6110/ssz_static/Withdrawal
+tests/minimal/eip6110/sync/optimistic
+tests/minimal/eip6110/transition/core
+
+# Whisk
+tests/mainnet/whisk/ssz_static/AggregateAndProof
+tests/mainnet/whisk/ssz_static/Attestation
+tests/mainnet/whisk/ssz_static/AttestationData
+tests/mainnet/whisk/ssz_static/AttesterSlashing
+tests/mainnet/whisk/ssz_static/BLSToExecutionChange
+tests/mainnet/whisk/ssz_static/BeaconBlock
+tests/mainnet/whisk/ssz_static/BeaconBlockBody
+tests/mainnet/whisk/ssz_static/BeaconBlockHeader
+tests/mainnet/whisk/ssz_static/BeaconState
+tests/mainnet/whisk/ssz_static/Checkpoint
+tests/mainnet/whisk/ssz_static/ContributionAndProof
+tests/mainnet/whisk/ssz_static/Deposit
+tests/mainnet/whisk/ssz_static/DepositData
+tests/mainnet/whisk/ssz_static/DepositMessage
+tests/mainnet/whisk/ssz_static/Eth1Block
+tests/mainnet/whisk/ssz_static/Eth1Data
+tests/mainnet/whisk/ssz_static/ExecutionPayload
+tests/mainnet/whisk/ssz_static/ExecutionPayloadHeader
+tests/mainnet/whisk/ssz_static/Fork
+tests/mainnet/whisk/ssz_static/ForkData
+tests/mainnet/whisk/ssz_static/HistoricalBatch
+tests/mainnet/whisk/ssz_static/HistoricalSummary
+tests/mainnet/whisk/ssz_static/IndexedAttestation
+tests/mainnet/whisk/ssz_static/LightClientBootstrap
+tests/mainnet/whisk/ssz_static/LightClientFinalityUpdate
+tests/mainnet/whisk/ssz_static/LightClientHeader
+tests/mainnet/whisk/ssz_static/LightClientOptimisticUpdate
+tests/mainnet/whisk/ssz_static/LightClientUpdate
+tests/mainnet/whisk/ssz_static/PendingAttestation
+tests/mainnet/whisk/ssz_static/PowBlock
+tests/mainnet/whisk/ssz_static/ProposerSlashing
+tests/mainnet/whisk/ssz_static/SignedAggregateAndProof
+tests/mainnet/whisk/ssz_static/SignedBLSToExecutionChange
+tests/mainnet/whisk/ssz_static/SignedBeaconBlock
+tests/mainnet/whisk/ssz_static/SignedBeaconBlockHeader
+tests/mainnet/whisk/ssz_static/SignedContributionAndProof
+tests/mainnet/whisk/ssz_static/SignedVoluntaryExit
+tests/mainnet/whisk/ssz_static/SigningData
+tests/mainnet/whisk/ssz_static/SyncAggregate
+tests/mainnet/whisk/ssz_static/SyncAggregatorSelectionData
+tests/mainnet/whisk/ssz_static/SyncCommittee
+tests/mainnet/whisk/ssz_static/SyncCommitteeContribution
+tests/mainnet/whisk/ssz_static/SyncCommitteeMessage
+tests/mainnet/whisk/ssz_static/Validator
+tests/mainnet/whisk/ssz_static/VoluntaryExit
+tests/mainnet/whisk/ssz_static/WhiskTracker
+tests/mainnet/whisk/ssz_static/Withdrawal
+tests/minimal/whisk/ssz_static/AggregateAndProof
+tests/minimal/whisk/ssz_static/Attestation
+tests/minimal/whisk/ssz_static/AttestationData
+tests/minimal/whisk/ssz_static/AttesterSlashing
+tests/minimal/whisk/ssz_static/BLSToExecutionChange
+tests/minimal/whisk/ssz_static/BeaconBlock
+tests/minimal/whisk/ssz_static/BeaconBlockBody
+tests/minimal/whisk/ssz_static/BeaconBlockHeader
+tests/minimal/whisk/ssz_static/BeaconState
+tests/minimal/whisk/ssz_static/Checkpoint
+tests/minimal/whisk/ssz_static/ContributionAndProof
+tests/minimal/whisk/ssz_static/Deposit
+tests/minimal/whisk/ssz_static/DepositData
+tests/minimal/whisk/ssz_static/DepositMessage
+tests/minimal/whisk/ssz_static/Eth1Block
+tests/minimal/whisk/ssz_static/Eth1Data
+tests/minimal/whisk/ssz_static/ExecutionPayload
+tests/minimal/whisk/ssz_static/ExecutionPayloadHeader
+tests/minimal/whisk/ssz_static/Fork
+tests/minimal/whisk/ssz_static/ForkData
+tests/minimal/whisk/ssz_static/HistoricalBatch
+tests/minimal/whisk/ssz_static/HistoricalSummary
+tests/minimal/whisk/ssz_static/IndexedAttestation
+tests/minimal/whisk/ssz_static/LightClientBootstrap
+tests/minimal/whisk/ssz_static/LightClientFinalityUpdate
+tests/minimal/whisk/ssz_static/LightClientHeader
+tests/minimal/whisk/ssz_static/LightClientOptimisticUpdate
+tests/minimal/whisk/ssz_static/LightClientUpdate
+tests/minimal/whisk/ssz_static/PendingAttestation
+tests/minimal/whisk/ssz_static/PowBlock
+tests/minimal/whisk/ssz_static/ProposerSlashing
+tests/minimal/whisk/ssz_static/SignedAggregateAndProof
+tests/minimal/whisk/ssz_static/SignedBLSToExecutionChange
+tests/minimal/whisk/ssz_static/SignedBeaconBlock
+tests/minimal/whisk/ssz_static/SignedBeaconBlockHeader
+tests/minimal/whisk/ssz_static/SignedContributionAndProof
+tests/minimal/whisk/ssz_static/SignedVoluntaryExit
+tests/minimal/whisk/ssz_static/SigningData
+tests/minimal/whisk/ssz_static/SyncAggregate
+tests/minimal/whisk/ssz_static/SyncAggregatorSelectionData
+tests/minimal/whisk/ssz_static/SyncCommittee
+tests/minimal/whisk/ssz_static/SyncCommitteeContribution
+tests/minimal/whisk/ssz_static/SyncCommitteeMessage
+tests/minimal/whisk/ssz_static/Validator
+tests/minimal/whisk/ssz_static/VoluntaryExit
+tests/minimal/whisk/ssz_static/WhiskTracker
+tests/minimal/whisk/ssz_static/Withdrawal