-
Notifications
You must be signed in to change notification settings - Fork 36
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore(levm): daily report levm tests & diff (#1668)
**Motivation** We want to compare the default feature build vs the one using levm, since we want to be aware of possible regressions while using said vm. Closes #1645. Closes #1627
- Loading branch information
Showing
6 changed files
with
171 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| #!/bin/bash | ||
|
|
||
| if [ "$#" -ne 2 ]; then | ||
| echo "Usage: $0 <revm_file> <levm_file>" | ||
| exit 1 | ||
| fi | ||
|
|
||
| revm_file=$1 | ||
| levm_file=$2 | ||
|
|
||
| # Check if files exist | ||
| if [ ! -f "$revm_file" ]; then | ||
| echo "Error: Revm file '$revm_file' not found" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ ! -f "$levm_file" ]; then | ||
| echo "Error: LEVM file '$levm_file' not found" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Create a temporary file | ||
| TEMP_FILE=$(mktemp) | ||
| trap 'rm -f $TEMP_FILE' EXIT | ||
|
|
||
| get_last_section() { | ||
| tac "$1" | sed -n "1,/\*Total:/p" | tac | ||
| } | ||
|
|
||
| parse_results() { | ||
| while IFS= read -r line; do | ||
| if [[ $line =~ ^[[:space:]]*[^*] && $line =~ : ]]; then | ||
| name=$(echo "$line" | cut -d':' -f1 | tr -d '\t' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') | ||
| values=$(echo "$line" | cut -d':' -f2 | tr -d ' ') | ||
| passed=$(echo "$values" | cut -d'/' -f1) | ||
| total=$(echo "$values" | cut -d'/' -f2 | cut -d'(' -f1) | ||
| percentage=$(echo "$values" | grep -o "[0-9.]*%" | tr -d '%') | ||
| echo "$name|$passed|$total|$percentage" | ||
| fi | ||
| done < <(get_last_section "$1") | ||
| } | ||
|
|
||
| revm_results=$(parse_results "$revm_file") | ||
| levm_results=$(parse_results "$levm_file") | ||
|
|
||
| found_differences=false | ||
|
|
||
| echo "$revm_results" > "$TEMP_FILE" | ||
|
|
||
| while IFS='|' read -r name revm_passed revm_total revm_percentage; do | ||
| if [ -n "$name" ]; then | ||
| levm_line=$(echo "$levm_results" | grep "^$name|" || true) | ||
| if [ -n "$levm_line" ]; then | ||
| levm_passed=$(echo "$levm_line" | cut -d'|' -f2) | ||
| levm_total=$(echo "$levm_line" | cut -d'|' -f3) | ||
| levm_percentage=$(echo "$levm_line" | cut -d'|' -f4) | ||
|
|
||
| if [ "$levm_passed" != "$revm_passed" ]; then | ||
| if [ "$found_differences" = false ]; then | ||
| echo "Found differences between LEVM and revm: :warning:" | ||
| echo | ||
| found_differences=true | ||
| fi | ||
| if [ "$levm_passed" -gt "$revm_passed" ]; then | ||
| echo "• *$name* (improvement :arrow_up:):" | ||
| else | ||
| echo "• *$name* (regression :arrow_down:):" | ||
| fi | ||
| echo " - Revm: $revm_passed/$revm_total ($revm_percentage%)" | ||
| echo " - LEVM: $levm_passed/$levm_total ($levm_percentage%)" | ||
| echo 1 >> "$TEMP_FILE.diff" | ||
| fi | ||
| else | ||
| if [ "$found_differences" = false ]; then | ||
| echo "Found differences between LEVM and revm: :warning:" | ||
| echo | ||
| found_differences=true | ||
| fi | ||
| echo "• *$name*: Test present in revm but missing in LEVM :x:" | ||
| echo 1 >> "$TEMP_FILE.diff" | ||
| fi | ||
| fi | ||
| done < "$TEMP_FILE" | ||
|
|
||
| if [ ! -f "$TEMP_FILE.diff" ]; then | ||
| echo "No differences found between revm and LEVM implementations! :white_check_mark:" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| curl -X POST $url \ | ||
| -H 'Content-Type: application/json; charset=utf-8' \ | ||
| --data @- <<EOF | ||
| $(jq -n --arg text "$(cat results_default.md)" '{ | ||
| "blocks": [ | ||
| { | ||
| "type": "header", | ||
| "text": { | ||
| "type": "plain_text", | ||
| "text": "Daily Hive Coverage report" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": $text | ||
| } | ||
| } | ||
| ] | ||
| }') | ||
| EOF |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| curl -X POST $url \ | ||
| -H 'Content-Type: application/json; charset=utf-8' \ | ||
| --data @- <<EOF | ||
| $(jq -n --arg text "$(cat diff.md)" '{ | ||
| "blocks": [ | ||
| { | ||
| "type": "header", | ||
| "text": { | ||
| "type": "plain_text", | ||
| "text": "Hive tests revm vs levm diff" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": $text | ||
| } | ||
| } | ||
| ] | ||
| }') | ||
| EOF |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters