Merge pull request #6063 from Agoric/mhofman/fix-process-integration-…

…results Fix process-integration-results.sh
Agoric · Aug 27, 2022 · 6494a03 · 6494a03
2 parents 8d5bd3f + 7b5b2e8
commit 6494a03
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 49 deletions.
diff --git a/scripts/process-integration-results.sh b/scripts/process-integration-results.sh
@@ -11,52 +11,20 @@ RESULTSDIR=${RESULTSDIR-"$NETWORK_NAME/results"}
 
 [ $# -gt 0 ] && RESULTSDIR="$1"
 
+clean_slog() {
+  jq -cr 'del(.time, .monotime, .dr[2].timestamps, .memoryUsage, .heapStats, .statsTime)'
+}
+
 "$thisdir/process-integration-swingstore-traces.sh" "$RESULTSDIR"
 
-[ -f "$RESULTSDIR/divergent_snapshots" ] || exit 0
+# [ -f "$RESULTSDIR/divergent_snapshots" ] || exit 0
 
 cd "$RESULTSDIR"
 
-mkdir -p "xs-snapshots"
-cp -a validator0-xs-snapshots/* "xs-snapshots/" || true
-cp -a validator1-xs-snapshots/* "xs-snapshots/" || true
-for s in chain-*-storage.tar.xz; do
-  [ -f "$s" ] || continue
-  tar -C "xs-snapshots/" -xJf $s --wildcards '**/xs-snapshots/*.gz' --transform='s/.*\///'
-done
-
 to_backup="divergent_snapshots divergent_snapshot_vats validator-swingstore-trace.diff"
-to_delete="xs-snapshots"
-
-snapshots=""
-for trace in chain-*-swingstore-trace validator*-swingstore-trace; do
-  [ -f "$trace" ] || continue
-  snapshots_dir=${trace%"-swingstore-trace"}-snapshots
-  mkdir -p $snapshots_dir
-  to_delete="$to_delete $snapshots_dir"
-  for v in $({ grep -E 'set local\.v[0-9]+\.lastSnapshot' $trace || true; } | cut -d ' ' -f 2 | cut -d '.' -f 2 | sort | uniq ); do
-    mkdir -p $snapshots_dir/$v
-    if grep -q -e "^$v\$" divergent_snapshot_vats; then
-      to_backup="$to_backup $snapshots_dir/$v"
-      v_divergent=1
-    else
-      v_divergent=0
-    fi
-    while read -r parsed; do
-      set $parsed
-      [ $v_divergent -eq 1 ] && snapshots="$snapshots $1"
-      ln -sf -T ../../xs-snapshots/$1 $snapshots_dir/$v/$2
-    done < <({ grep "set local.$v.lastSnapshot" $trace || true; } | \
-      cut -d ' ' -f 3 | \
-      jq -src '[.[] | [.startPos.itemCount, .snapshotID] ] | to_entries[] | [.value[1], " ", (1 + .key | tostring | length | if . >= 3 then "" else "0" * (3 - .) end), (1 + .key | tostring), "-", (.value[0] | tostring)] | join("")' \
-    )
-  done
-done
+to_delete=""
 
-gunzip -f xs-snapshots/*.gz || true
-to_backup="$to_backup $(for h in $snapshots $(<divergent_snapshots); do
-  [ -f xs-snapshots/$h ] && echo xs-snapshots/$h || true
-done | sort | uniq)"
+[ -f monitor-vs-validator-swingstore-trace.diff ] && to_backup="$to_backup monitor-vs-validator-swingstore-trace.diff"
 
 # TODO: handle vat suspension (aka same vatID, multiple workers)
 (mkdir -p validator0-xsnap-trace && cd $_ && tar -xzf ../$_.tgz && for v in *; do [ -d $v -a ! -h $v ] && ln -sf -T $v $(jq -r '.name | split(":") | .[0]' $v/00000-options.json) ; done; true)
@@ -131,17 +99,56 @@ for v in validator0-xsnap-trace/v*; do
   done
 done | grep -v "No newline at end of file" > monitor-vs-validator-xsnap-trace.diff || true
 [ "x${DEBUG-}" = "x1" ] && set -x
-to_backup="$to_backup monitor-vs-validator-xsnap-trace.diff"
+grep -e '^--- validator' <(cat validator-xsnap-trace.diff monitor-vs-validator-xsnap-trace.diff) | cut -d '/' -f 2 | uniq > divergent_xsnap_trace_vats || true
 
-[ -f monitor-vs-validator-swingstore-trace.diff ] && to_backup="$to_backup monitor-vs-validator-swingstore-trace.diff"
+to_backup="$to_backup monitor-vs-validator-xsnap-trace.diff divergent_xsnap_trace_vats"
+
+mkdir -p "xs-snapshots"
+cp -a validator0-xs-snapshots/* "xs-snapshots/" || true
+cp -a validator1-xs-snapshots/* "xs-snapshots/" || true
+for s in chain-*-storage.tar.xz; do
+  [ -f "$s" ] || continue
+  tar -C "xs-snapshots/" -xJf $s --wildcards '**/xs-snapshots/*.gz' --transform='s/.*\///'
+done
+to_delete="$to_delete xs-snapshots"
+
+snapshots=""
+for trace in chain-*-swingstore-trace validator*-swingstore-trace; do
+  [ -f "$trace" ] || continue
+  snapshots_dir=${trace%"-swingstore-trace"}-snapshots
+  mkdir -p $snapshots_dir
+  to_delete="$to_delete $snapshots_dir"
+  for v in $({ grep -E 'set local\.v[0-9]+\.lastSnapshot' $trace || true; } | cut -d ' ' -f 2 | cut -d '.' -f 2 | sort | uniq ); do
+    mkdir -p $snapshots_dir/$v
+    if grep -q -e "^$v\$" <(cat divergent_snapshot_vats divergent_xsnap_trace_vats); then
+      to_backup="$to_backup $snapshots_dir/$v"
+      v_divergent=1
+    else
+      v_divergent=0
+    fi
+    while read -r parsed; do
+      set $parsed
+      [ $v_divergent -eq 1 ] && snapshots="$snapshots $1"
+      ln -sf -T ../../xs-snapshots/$1 $snapshots_dir/$v/$2
+    done < <({ grep "set local.$v.lastSnapshot" $trace || true; } | \
+      cut -d ' ' -f 3 | \
+      jq -src '[.[] | [.startPos.itemCount, .snapshotID] ] | to_entries[] | [.value[1], " ", (1 + .key | tostring | length | if . >= 3 then "" else "0" * (3 - .) end), (1 + .key | tostring), "-", (.value[0] | tostring)] | join("")' \
+    )
+  done
+done
+
+gunzip -f xs-snapshots/*.gz || true
+to_backup="$to_backup $(for h in $snapshots $(<divergent_snapshots); do
+  [ -f xs-snapshots/$h ] && echo xs-snapshots/$h || true
+done | sort | uniq)"
 
 for trace in *-xsnap-trace; do
   [ -d "$trace" ] || continue
   snapshots_dir=${trace%"-xsnap-trace"}-snapshots
   for v in $trace/v*; do
     [ -h "$v" ] || continue
     v=${v#"$trace/"}
-    if grep -q -e "^$v\$" divergent_snapshot_vats; then
+    if grep -q -e "^$v\$" <(cat divergent_snapshot_vats divergent_xsnap_trace_vats); then
       v_divergent=1
       to_backup="$to_backup $trace/$v $trace/$(readlink $trace/$v)"
     else
@@ -172,14 +179,14 @@ for trace in *-xsnap-trace; do
   done
 done
 
-diff <(cat validator0.slog | jq -cr 'del(.time)') <(cat validator1.slog | jq -cr 'del(.time)') > validator-slog.diff || true
+diff <(cat validator0.slog | clean_slog) <(cat validator1.slog | clean_slog) > validator-slog.diff || true
 to_backup="$to_backup validator-slog.diff"
 
 if [ -f chain-stage-0.slog.gz ]; then
-  gunzip -fk chain-stage-0.slog.gz
+  zcat chain-stage-0.slog.gz > chain-stage-0.slog || true
   to_delete="$to_delete chain-stage-0.slog"
   chain_slog_len=$(cat chain-stage-0.slog | wc -l)
-  diff <(head -n $chain_slog_len validator0.slog | jq -cr 'del(.time)') <(cat chain-stage-0.slog | jq -cr 'del(.time)') > monitor-stage-0-vs-validator-slog.diff || true
+  diff <(head -n $chain_slog_len validator0.slog | clean_slog) <(cat chain-stage-0.slog | clean_slog) > monitor-stage-0-vs-validator-slog.diff || true
   to_backup="$to_backup monitor-stage-0-vs-validator-slog.diff"
 fi
 

diff --git a/scripts/process-integration-swingstore-traces.sh b/scripts/process-integration-swingstore-traces.sh
@@ -9,15 +9,15 @@ RESULTSDIR=${RESULTSDIR-"$NETWORK_NAME/results"}
 [ $# -gt 0 ] && RESULTSDIR="$1"
 
 get_trace_len() {
-  grep -n commit-tx $1 | tail -1 | cut -d : -f 1 || echo 0
+  grep -n commit-tx $1 | tail -1 | cut -d : -f 1 || cat $1 | wc -l || echo 0
 }
 
 get_vats_from_diff() {
-  grep 'set local\.v' $1 | cut -d . -f 2 | sort | uniq
+  grep 'set local\.v' $1 | cut -d . -f 2 | sort | uniq || true
 }
 
 get_snapshots_from_diff() {
-  grep 'set local\.snapshot\.' $1 | cut -d ' ' -f 2 | cut -d . -f 3 | sort | uniq
+  grep 'set local\.snapshot\.' $1 | cut -d ' ' -f 2 | cut -d . -f 3 | sort | uniq || true
 }
 
 val0len=$(get_trace_len $RESULTSDIR/validator0-swingstore-trace)
@@ -46,7 +46,7 @@ if [ -f $RESULTSDIR/chain-stage-0-swingstore-trace ] ; then
   diffs="$diffs $RESULTSDIR/monitor-vs-validator-swingstore-trace.diff"
 fi
 
-[ $val_diff_ret -eq 0 -a $chain_diff_ret -eq 0 ] && exit 0
+# [ $val_diff_ret -eq 0 -a $chain_diff_ret -eq 0 ] && exit 0
 
 get_snapshots_from_diff <(cat $diffs) > $RESULTSDIR/divergent_snapshots
 get_vats_from_diff <(cat $diffs) > $RESULTSDIR/divergent_snapshot_vats