chore: improve e2e and pipeline for debuggability (#4552)

Author: geyslan

pkg/ebpf/events_pipeline.go

@@ -222,7 +222,14 @@ func (t *Tracee) decodeEvents(ctx context.Context, sourceChan chan []byte) (<-ch
 					syscall = events.Core.GetDefinitionByID(id).GetName()
 				} else {
 					// This should never fail, as the translation used in eBPF relies on the same event definitions
-					logger.Errorw(fmt.Sprintf("No syscall event with id %d", id))
+					commStr := string(eCtx.Comm[:bytes.IndexByte(eCtx.Comm[:], 0)])
+					utsNameStr := string(eCtx.UtsName[:bytes.IndexByte(eCtx.UtsName[:], 0)])
+					logger.Errorw(
+						fmt.Sprintf("Event %s with an invalid syscall id %d", evtName, id),
+						"Comm", commStr,
+						"UtsName", utsNameStr,
+						"EventContext", eCtx,
+					)
 				}
 			}
 

tests/e2e-inst-test.sh

@@ -79,6 +79,9 @@ if [[ ! -x ./dist/tracee ]]; then
     error_exit "could not find tracee executable"
 fi
 
+logfile=$SCRIPT_TMP_DIR/tracee-log-$$
+outputfile=$SCRIPT_TMP_DIR/output-$$
+
 anyerror=""
 
 # Run tests, one by one
@@ -138,18 +141,18 @@ for TEST in $TESTS; do
 
     # Run tracee
 
-    rm -f $SCRIPT_TMP_DIR/build-$$
-    rm -f $SCRIPT_TMP_DIR/tracee-log-$$
+    rm -f $outputfile
+    rm -f $logfile
 
     tracee_command="./dist/tracee \
                         --install-path $TRACEE_TMP_DIR \
                         --cache cache-type=mem \
                         --cache mem-cache-size=512 \
                         --proctree source=both \
                         --output option:sort-events \
-                        --output json:$SCRIPT_TMP_DIR/build-$$ \
                         --output option:parse-arguments \
-                        --log file:$SCRIPT_TMP_DIR/tracee-log-$$ \
+                        --output json:$outputfile \
+                        --log file:$logfile \
                         --signatures-dir "$SIG_DIR" \
                         --dnscache enable \
                         --grpc-listen-addr unix:/tmp/tracee.sock \
@@ -201,7 +204,7 @@ for TEST in $TESTS; do
         info
         info "$TEST: FAILED. ERRORS:"
         info
-        cat $SCRIPT_TMP_DIR/tracee-log-$$
+        cat $logfile
 
         anyerror="${anyerror}$TEST,"
         continue
@@ -232,12 +235,18 @@ for TEST in $TESTS; do
 
     # The cleanup happens at EXIT
 
-    logfile=$SCRIPT_TMP_DIR/tracee-log-$$
+    # Make sure we exit tracee before checking output and log files
+
+    pid_tracee=$(pidof tracee | cut -d' ' -f1)
+    kill -SIGINT "$pid_tracee"
+    sleep $TRACEE_SHUTDOWN_TIMEOUT
+    kill -SIGKILL "$pid_tracee" >/dev/null 2>&1
+    sleep 3
 
     # Check if the test has failed or not
 
     found=0
-    cat $SCRIPT_TMP_DIR/build-$$ | jq .eventName | grep -q "$TEST" && found=1
+    cat $outputfile | jq .eventName | grep -q "$TEST" && found=1
     errors=$(cat $logfile | wc -l 2>/dev/null)
 
     if [[ $TEST == "BPF_ATTACH" ]]; then
@@ -249,25 +258,33 @@ for TEST in $TESTS; do
         info "$TEST: SUCCESS"
     else
         anyerror="${anyerror}$TEST,"
+
         info "$TEST: FAILED, stderr from tracee:"
-        cat $SCRIPT_TMP_DIR/tracee-log-$$
+        cat $logfile
+
         info "$TEST: FAILED, events from tracee:"
-        cat $SCRIPT_TMP_DIR/build-$$
+        cat $outputfile
+
+        info "Tracee command:"
+        echo "$tracee_command" | tr -s ' '
+
+        info "Tracee process is running?"
+        traceepids=$(pgrep tracee)
+        if [[ -n $traceepids ]]; then
+            info "YES, Tracee is still running (should not be, fix me!), pids: $traceepids"
+            info "Aborting tests"
+            break
+        else
+            info "NO, Tracee is not running"
+        fi
         info
     fi
     info
 
-    rm -f $SCRIPT_TMP_DIR/build-$$
-    rm -f $SCRIPT_TMP_DIR/tracee-log-$$
-
-    # Make sure we exit tracee to start it again
-
-    pid_tracee=$(pidof tracee | cut -d' ' -f1)
-    kill -SIGINT "$pid_tracee"
-    sleep $TRACEE_SHUTDOWN_TIMEOUT
-    kill -SIGKILL "$pid_tracee" >/dev/null 2>&1
-    sleep 3
+    # Cleanup
 
+    rm -f $outputfile
+    rm -f $logfile
     # Cleanup leftovers
     rm -rf $TRACEE_TMP_DIR
 done