diff --git a/.github/scripts/levm_revm_diff.sh b/.github/scripts/levm_revm_diff.sh new file mode 100644 index 0000000000..5c676d8657 --- /dev/null +++ b/.github/scripts/levm_revm_diff.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +revm_file=$1 +levm_file=$2 + +# Check if files exist +if [ ! -f "$revm_file" ]; then + echo "Error: Revm file '$revm_file' not found" + exit 1 +fi + +if [ ! -f "$levm_file" ]; then + echo "Error: LEVM file '$levm_file' not found" + exit 1 +fi + +# Create a temporary file +TEMP_FILE=$(mktemp) +trap 'rm -f $TEMP_FILE' EXIT + +get_last_section() { + tac "$1" | sed -n "1,/\*Total:/p" | tac +} + +parse_results() { + while IFS= read -r line; do + if [[ $line =~ ^[[:space:]]*[^*] && $line =~ : ]]; then + name=$(echo "$line" | cut -d':' -f1 | tr -d '\t' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + values=$(echo "$line" | cut -d':' -f2 | tr -d ' ') + passed=$(echo "$values" | cut -d'/' -f1) + total=$(echo "$values" | cut -d'/' -f2 | cut -d'(' -f1) + percentage=$(echo "$values" | grep -o "[0-9.]*%" | tr -d '%') + echo "$name|$passed|$total|$percentage" + fi + done < <(get_last_section "$1") +} + +revm_results=$(parse_results "$revm_file") +levm_results=$(parse_results "$levm_file") + +found_differences=false + +echo "$revm_results" > "$TEMP_FILE" + +while IFS='|' read -r name revm_passed revm_total revm_percentage; do + if [ -n "$name" ]; then + levm_line=$(echo "$levm_results" | grep "^$name|" || true) + if [ -n "$levm_line" ]; then + levm_passed=$(echo "$levm_line" | cut -d'|' -f2) + levm_total=$(echo "$levm_line" | cut -d'|' -f3) + levm_percentage=$(echo "$levm_line" | cut -d'|' -f4) + + if [ "$levm_passed" != "$revm_passed" ]; then + if [ "$found_differences" = false ]; then + echo "Found differences between LEVM and revm: :warning:" + echo + found_differences=true + fi + if [ "$levm_passed" -gt "$revm_passed" ]; then + echo "• *$name* (improvement :arrow_up:):" + else + echo "• *$name* (regression :arrow_down:):" + fi + echo " - Revm: $revm_passed/$revm_total ($revm_percentage%)" + echo " - LEVM: $levm_passed/$levm_total ($levm_percentage%)" + echo 1 >> "$TEMP_FILE.diff" + fi + else + if [ "$found_differences" = false ]; then + echo "Found differences between LEVM and revm: :warning:" + echo + found_differences=true + fi + echo "• *$name*: Test present in revm but missing in LEVM :x:" + echo 1 >> "$TEMP_FILE.diff" + fi + fi +done < "$TEMP_FILE" + +if [ ! -f "$TEMP_FILE.diff" ]; then + echo "No differences found between revm and LEVM implementations! :white_check_mark:" +fi diff --git a/.github/scripts/publish.sh b/.github/scripts/publish.sh index ef140e3b06..63164b1ef8 100644 --- a/.github/scripts/publish.sh +++ b/.github/scripts/publish.sh @@ -1,7 +1,7 @@ curl -X POST $url \ -H 'Content-Type: application/json; charset=utf-8' \ --data @- < results.md + - name: Upload ${{matrix.vm}} result + uses: actions/upload-artifact@v4 + with: + name: results_${{matrix.vm}}.md + path: results.md + if-no-files-found: error + - name: Post results in summary run: | echo "# Hive coverage report (${{ matrix.vm }})" >> $GITHUB_STEP_SUMMARY @@ -114,6 +122,32 @@ jobs: sh .github/scripts/publish_levm_hive.sh fi + hive-diff-report: + name: Post tests diff to levm slack + needs: hive-report + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Download results (levm) + uses: actions/download-artifact@v4 + with: + name: results_levm.md + + - name: Download results (revm) + uses: actions/download-artifact@v4 + with: + name: results_revm.md + + - name: Post results diff in LEVM channel + env: + url: ${{ secrets.LEVM_SLACK_WEBHOOK }} + run: | + bash .github/scripts/levm_revm_diff.sh results_revm.md results_levm.md >> diff.md + cat diff.md >> $GITHUB_STEP_SUMMARY + bash .github/scripts/publish_revm_levm_diff.sh + levm-test: name: Generate Report for LEVM EF Tests runs-on: ubuntu-latest @@ -144,10 +178,10 @@ jobs: echo "# Daily LEVM EF Tests Run Report" >> $GITHUB_STEP_SUMMARY cat cmd/ef_tests/levm/levm_ef_tests_summary_github.txt >> $GITHUB_STEP_SUMMARY - - name: Post results to ethrex L2 slack channel - env: - url: ${{ secrets.ETHREX_L2_SLACK_WEBHOOK }} - run: sh .github/scripts/publish_levm_ef_tests_summary.sh + # - name: Post results to ethrex L2 slack channel + # env: + # url: ${{ secrets.ETHREX_L2_SLACK_WEBHOOK }} + # run: sh .github/scripts/publish_levm_ef_tests_summary.sh - name: Post results to levm slack channel env: