Multiple bug fixes

.github/workflows/default_install.yml

@@ -41,7 +41,7 @@ jobs:
             wget https://ftp.dlink.de/dir/dir-300/archive/driver_software/DIR-300_fw_revb_214b01_ALL_de_20130206.zip
       - name: EMBA default analysis
         run: |
-            sudo ./emba -f ./DIR-300_fw_revb_214b01_ALL_de_20130206.zip -l ./logs_emba -S -p ./scan-profiles/default-scan-emulation.emba -y -j
+            sudo ./emba -f ./DIR-300_fw_revb_214b01_ALL_de_20130206.zip -l ./logs_emba -S -p ./scan-profiles/default-scan.emba -y -j
       - name: Check result files exist
         id: check_files
         uses: andstor/file-existence-action@v2

config/linux_common_files.txt

@@ -177046,7 +177046,6 @@ sof-hda-generic.tplg
 sof-hsw.ldc
 sof-hsw.ri
 sof-hsw-rt5640.tplg
-Sofia
 sof-icl-dmic-4ch.tplg
 sof-icl.ldc
 sof-icl-nocodec.tplg

emba

@@ -480,6 +480,13 @@ main() {
       echo "GID: $(id -g "${SUDO_USER:-${USER}}")" >> "${LOG_DIR}"/orig_user.log
     fi
 
+    if [[ "${IN_DOCKER}" -eq 0 ]]; then
+      print_running_modules &
+      RUN_MOD_PID="$!"
+      store_kill_pids "${RUN_MOD_PID}"
+      disown "${RUN_MOD_PID}" 2> /dev/null || true
+    fi
+
     # Print additional information about the firmware (-Y, -X, -Z, -N)
     print_firmware_info "${FW_VENDOR}" "${FW_VERSION}" "${FW_DEVICE}" "${FW_NOTES}"
     if [[ "${KERNEL}" -ne 1 ]] && [[ "${CONTAINER_EXTRACT}" -ne 1 ]] && [[ "${ONLY_DEP}" -eq 0 ]]; then
@@ -749,9 +756,9 @@ main() {
     print_ln "no_log"
 
     if [[ -d "${LOG_DIR}" ]]; then
-      print_output "[!] Pre-checking phase ended on ""$(date)"" and took about ""$(date -d@"${SECONDS}" -u +%H:%M:%S)"" \\n" "main" 
+      print_output "[!] Pre-checking phase ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "main"
     else
-      print_output "[!] Pre-checking phase ended on ""$(date)"" and took about ""$(date -d@"${SECONDS}" -u +%H:%M:%S)"" \\n" "no_log"
+      print_output "[!] Pre-checking phase ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "no_log"
     fi
     write_notification "Pre-checking phase finished"
 
@@ -785,9 +792,9 @@ main() {
     print_ln "no_log"
 
     if [[ -d "${LOG_DIR}" ]]; then
-      print_output "[!] Testing phase ended on ""$(date)"" and took about ""$(date -d@"${SECONDS}" -u +%H:%M:%S)"" \\n" "main"
+      print_output "[!] Testing phase ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "main"
     else
-      print_output "[!] Testing phase ended on ""$(date)"" and took about ""$(date -d@"${SECONDS}" -u +%H:%M:%S)"" \\n" "no_log"
+      print_output "[!] Testing phase ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "no_log"
     fi
     write_notification "Testing phase ended"
 
@@ -812,9 +819,9 @@ main() {
 
     print_ln "no_log"
     if [[ -d "${LOG_DIR}" ]]; then
-      print_output "[!] System emulation phase ended on ""$(date)"" and took about ""$(date -d@"${SECONDS}" -u +%H:%M:%S)"" \\n" "main"
+      print_output "[!] System emulation phase ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "main"
     else
-      print_output "[!] System emulation ended on ""$(date)"" and took about ""$(date -d@"${SECONDS}" -u +%H:%M:%S)"" \\n" "no_log"
+      print_output "[!] System emulation ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "no_log"
     fi
     write_notification "System emulation phase ended"
   fi
@@ -846,14 +853,14 @@ main() {
     fi
     print_ln "no_log"
     if [[ -d "${LOG_DIR}" ]]; then
-      print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"${SECONDS}" -u +%H:%M:%S)"" \\n" "main" 
+      print_output "[!] Test ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "main"
       write_notification "EMBA finished analysis"
       rm -r "${TMP_DIR}" 2>/dev/null || true
     else
-      print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"${SECONDS}" -u +%H:%M:%S)"" \\n" "no_log"
+      print_output "[!] Test ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "no_log"
     fi
     write_grep_log "$(date)" "TIMESTAMP"
-    write_grep_log "$(date -d@"${SECONDS}" -u +%H:%M:%S)" "DURATION"
+    write_grep_log "$(date -d@"${SECONDS}" -u +%d:%H:%M:%S)" "DURATION"
   else
     print_output "[!] No extracted firmware found" "no_log"
     print_output "$(indent "Try using binwalk or something else to extract the firmware")"

helpers/helpers_emba_helpers.sh

@@ -174,7 +174,7 @@ cleaner() {
   fi
   export CLEANED=1
   if [[ "$INTERRUPT_CLEAN" -eq 1 ]]; then
-    print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"$SECONDS" -u +%H:%M:%S)"" \\n" "no_log"
+    print_output "[!] Test ended on ""$(date)"" and took about ""$(show_runtime)"" \\n" "no_log"
     exit 1
   fi
 }

helpers/helpers_emba_html_generator.sh

@@ -92,6 +92,7 @@ add_link_tags() {
     readarray -t REF_LINKS_L_NUMBER < <(grep -a -n -E '\[REF\].*' "$LINK_FILE" | cut -d':' -f1 )
     for REF_LINK_NUMBER in "${REF_LINKS_L_NUMBER[@]}" ; do
       REF_LINK="$(sed "$REF_LINK_NUMBER""q;d" "$LINK_FILE" | cut -c12- | cut -d'<' -f1 || true)"
+      URL_REGEX='(www.|https?|ftp|file):\/\/'
       if [[ -f "$(echo "$REF_LINK" | cut -d"#" -f1)" ]] ; then
         if [[  ( ("${REF_LINK: -4}" == ".txt") || ("${REF_LINK: -4}" == ".log") ) || ( ("$REF_LINK" == *".txt#"*) || ("$REF_LINK" == *".log#"*) ) ]] ; then
           REF_ANCHOR=""
@@ -150,7 +151,6 @@ add_link_tags() {
           done
           LINK_COMMAND_ARR+=( "$LINE_NUMBER_INFO_PREV"'s@^@'"$HTML_LINK"'@' "$LINE_NUMBER_INFO_PREV"'s@$@'"$LINK_END"'@')
         fi
-      URL_REGEX='(www.|https?|ftp|file):\/\/'
       elif [[ "$REF_LINK" =~ $URL_REGEX ]] ; then
         LINE_NUMBER_INFO_PREV="$(( REF_LINK_NUMBER - 1 ))"
         while [[ ("$(sed "$LINE_NUMBER_INFO_PREV""q;d" "$LINK_FILE")" == "$P_START$SPAN_END$P_END") || ("$(sed "$LINE_NUMBER_INFO_PREV""q;d" "$LINK_FILE")" == "$BR" ) ]] ; do 
@@ -269,7 +269,7 @@ add_link_tags() {
       for SNYK_KEY in "${SNYK_KEY_F[@]}" ; do
         SNYK_ID_LINE="$(echo "$SNYK_KEY" | cut -d ":" -f 1)"
         SNYK_ID_STRING="$(echo "$SNYK_KEY" | cut -d ":" -f 2-)"
-        readarray -t SNYK_KEY_STRING_ARR < <(echo "$SNYK_ID_STRING" | tr " " "\n" | grep "SNYK-" | uniq)
+        readarray -t SNYK_KEY_STRING_ARR < <(echo "$SNYK_ID_STRING" | tr " " "\n" | grep "SNYK-" | uniq || true)
         for SNYK_KEY_ELEM in "${SNYK_KEY_STRING_ARR[@]}" ; do
           HTML_LINK="$(echo "$SNYK_LINK" | sed -e "s@LINKNAME@$SNYK_KEY_ELEM@g" | sed -e "s@LINK@$SNYK_KEY_ELEM@g")""$SNYK_KEY_ELEM""$LINK_END"
           LINK_COMMAND_ARR+=( "$SNYK_ID_LINE"'s@'"$SNYK_KEY_ELEM"'@'"$HTML_LINK"'@' )
@@ -282,7 +282,7 @@ add_link_tags() {
       for PSS_KEY in "${PSS_KEY_F[@]}" ; do
         PSS_ID_LINE="$(echo "$PSS_KEY" | cut -d ":" -f 1)"
         PSS_ID_STRING="$(echo "$PSS_KEY" | cut -d ":" -f 2-)"
-        readarray -t PSS_KEY_STRING_ARR < <(echo "$PSS_ID_STRING" | tr " " "\n" | grep -E "[0-9]+/.*\.html" | uniq)
+        readarray -t PSS_KEY_STRING_ARR < <(echo "$PSS_ID_STRING" | tr " " "\n" | grep -E "[0-9]+/.*\.html" | uniq || true)
         for PSS_KEY_NAME in "${PSS_KEY_STRING_ARR[@]}" ; do
           # PSS_KEY_NAME="$(echo "$PSS_KEY_ELEM" | tr "/" "_")"
           HTML_LINK="$(echo "$PSS_LINK" | sed -e "s@LINKNAME@$PSS_KEY_NAME@g" | sed -e "s@LINK@$PSS_KEY_NAME@g")""$PSS_KEY_NAME""$LINK_END"

helpers/helpers_emba_print.sh

@@ -832,3 +832,23 @@ print_notification() {
     fi
   done
 }
+
+print_running_modules() {
+  sleep 1h
+  while true; do
+    local STARTED_EMBA_PROCESSES=()
+    local EMBA_STARTED_PROC=""
+    mapfile -t STARTED_EMBA_PROCESSES < <(grep starting "${LOG_DIR}""/""${MAIN_LOG_FILE}" | awk '{print $9}'|| true)
+
+    for EMBA_STARTED_PROC in "${STARTED_EMBA_PROCESSES[@]}"; do
+      if ! grep -i -q "${EMBA_STARTED_PROC}"" finished" "${LOG_DIR}""/""${MAIN_LOG_FILE}"; then
+        print_output "[*] EMBA module ${ORANGE}${EMBA_STARTED_PROC}${NC} currently running" "no_log"
+      fi
+    done
+    sleep 1h
+  done
+}
+
+show_runtime() {
+  date -ud "@$SECONDS" +"$(( SECONDS/3600/24 )) days and %H:%M:%S"
+}

helpers/helpers_emba_system_emulation.sh

@@ -19,6 +19,7 @@ restart_emulation() {
   local IMAGE_NAME_="${2:-}"
   # restart_scan is used to indicate a restarted scan. For this we do not need to restart the network
   local RESTART_SCAN="${3:-0}"
+  local STATE_CHECK="${4:-"PING"}"
 
   if ping -c 1 "$IP_ADDRESS_" &> /dev/null; then
     print_output "[+] System with $ORANGE$IP_ADDRESS_$GREEN responding again - probably it recovered automatically.$NC"
@@ -43,7 +44,46 @@ restart_emulation() {
   ./run.sh &
   cd "$HOME_PATH" || (print_output "[-] EMBA path not available?")
 
-  COUNTER=0
+  if [[ "$STATE_CHECK" == "PING" ]]; then
+    ping_check "${IP_ADDRESS_}"
+  elif [[ "$STATE_CHECK" == "HPING" ]]; then
+    hping_check "${IP_ADDRESS_}"
+  elif [[ "$STATE_CHECK" == "TCP" ]]; then
+    # local PORT=80
+    print_output "[-] Check currently not implemented!"
+    # tcp_check "${IP_ADDRESS_}" "${PORT}"
+  fi
+}
+
+ping_check() {
+  local IP_ADDRESS_="${1:-}"
+  local COUNTER=0
+
+  while ! [[ "$(hping3 -n -c 1 "$IP_ADDRESS_" 2> /dev/null | grep -c "^len=")" -gt 0 ]]; do
+    print_output "[*] Waiting for restarted system ..."
+    ((COUNTER+=1))
+    if [[ "$COUNTER" -gt 50 ]]; then
+      print_output "[-] System not recovered"
+      break
+    fi
+    sleep 6
+  done
+
+  if [[ "$(hping3 -n -c 1 "$IP_ADDRESS_" 2>/dev/null | grep -c "^len=")" -gt 0 ]]; then
+    print_output "[*] System automatically maintained and should be available again in a few moments ... check ip address $ORANGE$IP_ADDRESS_$NC"
+    sleep 60
+    export SYS_ONLINE=1
+    export TCP="ok"
+  else
+    export SYS_ONLINE=0
+    export TCP="not ok"
+  fi
+}
+
+hping_check() {
+  local IP_ADDRESS_="${1:-}"
+  local COUNTER=0
+
   while ! ping -c 1 "$IP_ADDRESS_" &> /dev/null; do
     print_output "[*] Waiting for restarted system ..."
     ((COUNTER+=1))

installer/ID1_ubuntu_os.sh

@@ -42,18 +42,6 @@ ID1_ubuntu_os() {
       echo "Exec=/usr/lib/notification-daemon/notification-daemon" >> /usr/share/dbus-1/services/org.freedesktop.Notifications.service
     fi
 
-    if ! dpkg -l libssl1.1 &>/dev/null; then
-        # libssl1.1 missing
-        echo -e "\\n""$BOLD""Installing libssl1.1 for mongodb!""$NC"
-        # echo "deb http://security.ubuntu.com/ubuntu impish-security main" | tee /etc/apt/sources.list.d/impish-security.list
-        wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1-1ubuntu2.1~18.04.22_amd64.deb -O external/libssl-dev_1.1.1-1ubuntu2.1~18.04.22_amd64.deb
-        wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.22_amd64.deb -O external/libssl1.1_1.1.1-1ubuntu2.1~18.04.22_amd64.deb
-        dpkg -i external/libssl1.1_1.1.1-1ubuntu2.1~18.04.22_amd64.deb
-        dpkg -i external/libssl-dev_1.1.1-1ubuntu2.1~18.04.22_amd64.deb
-        rm external/libssl1.1_1.1.1-1ubuntu2.1~18.04.22_amd64.deb
-        rm external/libssl-dev_1.1.1-1ubuntu2.1~18.04.22_amd64.deb
-    fi
-
     if [[ "$WSL" -eq 1 ]]; then
       # docker installation on Ubuntu jammy in WSL environment is somehow broken
       echo -e "\\n""$MAGENTA""$BOLD""Docker installation for Ubuntu:jammy in WSL environment!""$NC"

installer/IF20_cve_search.sh

@@ -114,7 +114,35 @@ IF20_cve_search() {
         else
             echo -e "\\n""$MAGENTA""cve-search database not ready.""$NC"
         fi
+
+        cd "$HOME_PATH" || ( echo "Could not install EMBA component cve-search" && exit 1 )
         if [[ "$CVE_INST" -eq 1 ]]; then
+          if ! dpkg -s libssl1.1 &>/dev/null; then
+            # libssl1.1 missing
+            echo -e "\\n""$BOLD""Installing libssl1.1 for mongodb!""$NC"
+            # echo "deb http://security.ubuntu.com/ubuntu impish-security main" | tee /etc/apt/sources.list.d/impish-security.list
+            for i in {21..29}; do
+              echo "Testing download of libssl package version libssl1.1_1.1.1-1ubuntu2.1~18.04.${i}_amd64.deb"
+              wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1-1ubuntu2.1~18.04."${i}"_amd64.deb -O external/libssl-dev.deb || true
+                # http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1-1ubuntu2.1~18.04.22_amd64.deb
+              wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04."${i}"_amd64.deb -O external/libssl.deb || true
+                # http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.22_amd64.deb
+              if [[ "$(file external/libssl.deb)" == *"Debian binary package (format 2.0), with control.tar.xz, data compression xz"* ]]; then
+                break
+              else
+                [[ -f external/libssl.deb ]] && rm external/libssl.deb
+                [[ -f external/libssl-dev.deb ]] && rm external/libssl-dev.deb
+              fi
+            done
+
+            ! [[ -f external/libssl.deb ]] && ( "Could not install libssl" && exit 1)
+            ! [[ -f external/libssl-dev.deb ]] && ( "Could not install libssl-dev" && exit 1)
+            dpkg -i external/libssl.deb
+            dpkg -i external/libssl-dev.deb
+            [[ -f external/libssl.deb ]] && rm external/libssl.deb
+            [[ -f external/libssl-dev.deb ]] && rm external/libssl-dev.deb
+          fi
+
           wget --no-check-certificate -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/mongodb.gpg > /dev/null
           echo "deb [ signed-by=/etc/apt/trusted.gpg.d/mongodb.gpg ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list
           apt-get update -y
@@ -135,6 +163,7 @@ IF20_cve_search() {
             mongod --config /etc/mongod.conf &
           fi
 
+          cd ./external/cve-search/ || ( echo "Could not install EMBA component cve-search" && exit 1 )
           echo -e "\\n""$MAGENTA""$BOLD""The cve-search database will be downloaded and updated!""$NC"
           CVE_INST=1
           echo -e "\\n""$MAGENTA""Check if the cve-search database is already installed and populated.""$NC"