Merge pull request #1484 from m-1-k-3/s115_#1476

Foscam extraction (P20) fixes

docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   # nosemgrep
   emba:
-    image: embeddedanalyzer/emba:1.5.1d
+    image: embeddedanalyzer/emba:1.5.1f
     container_name: emba
     read_only: false
     # all pre-checker mount modules need privileged mode
@@ -51,7 +51,7 @@ services:
         soft: 0
 
   emba_quest:
-    image: embeddedanalyzer/emba:1.5.1d
+    image: embeddedanalyzer/emba:1.5.1f
     container_name: emba_quest
     read_only: true
     tmpfs:

helpers/helpers_emba_dependency_check.sh

@@ -134,7 +134,7 @@ check_nvd_db() {
   local lREMOTE_HASH="${1:-}"
   local lLOCAL_HASH=""
   if [[ -d "${EXT_DIR}"/nvd-json-data-feeds ]] ; then
-    lLOCAL_HASH="$(head "${EXT_DIR}"/nvd-json-data-feeds/.git/refs/heads/main)"
+    [[ -f "${EXT_DIR}"/nvd-json-data-feeds/.git/refs/heads/main ]] && lLOCAL_HASH="$(head "${EXT_DIR}"/nvd-json-data-feeds/.git/refs/heads/main)"
 
     if [[ "${lREMOTE_HASH}" == "${lLOCAL_HASH}" ]]; then
       echo -e "    CVE database version - ${GREEN}ok${NC}"
@@ -148,7 +148,7 @@ check_epss_db() {
   local lREMOTE_HASH="${1:-}"
   local lLOCAL_HASH=""
   if [[ -d "${EXT_DIR}"/EPSS-data ]] ; then
-    lLOCAL_HASH="$(head "${EXT_DIR}"/EPSS-data/.git/refs/heads/main)"
+    [[ -f "${EXT_DIR}"/EPSS-data/.git/refs/heads/main ]] && lLOCAL_HASH="$(head "${EXT_DIR}"/EPSS-data/.git/refs/heads/main)"
 
     if [[ "${lREMOTE_HASH}" == "${lLOCAL_HASH}" ]]; then
       echo -e "    EPSS database version - ${GREEN}ok${NC}"
@@ -162,7 +162,7 @@ check_git_hash() {
   local lREMOTE_HASH="${1:-}"
   local lLOCAL_HASH=""
   if git rev-parse --is-inside-work-tree >/dev/null 2>&1 ; then
-    lLOCAL_HASH="$(head .git/refs/heads/master)"
+    [[ -f .git/refs/heads/master ]] && lLOCAL_HASH="$(head .git/refs/heads/master)"
     # lLOCAL_HASH="$(git describe --always)"
 
     if [[ "${lREMOTE_HASH}" == "${lLOCAL_HASH}" ]]; then

helpers/helpers_emba_status_bar.sh

@@ -295,7 +295,7 @@ update_box_modules() {
     lBOX_SIZE="$(sed '1q;d' "${STATUS_TMP_PATH}" 2> /dev/null || true)"
   fi
   while [[ "${lBOX_SIZE}" -gt 0 ]]; do
-    [[ -f "${TMP_DIR}""/LINES.log" ]] && lLINES=$(cat "${TMP_DIR}""/LINES.log")
+    [[ -f "${TMP_DIR}""/LINES.log" ]] && lLINES=$(cat "${TMP_DIR}""/LINES.log" || true)
     lSTARTED_MODULE_STR="$(grep -c "starting\|blacklist triggered" "${LOG_DIR}/emba.log" 2> /dev/null || true )"
     lFINISHED_MODULE_STR="$(grep "finished\|blacklist triggered" "${LOG_DIR}/emba.log" 2> /dev/null | grep -vc "Quest container finished" || true )"
     lLAST_FINISHED_MODULE_STR="$(grep "finished" "${LOG_DIR}/emba.log" 2> /dev/null | grep -v "Quest container finished"| tail -1 | awk '{print $9}' | cut -d"_" -f1 || true )"

installer/IL15_emulated_checks_init.sh

@@ -52,6 +52,13 @@ IL15_emulated_checks_init() {
     # needed for cutycapt
     #
     # future extension
+    print_tool_info "libxslt1-dev"
+    print_tool_info "libxml2-dev"
+    # currently upnpclient failes during installing lxml:
+    #   ERROR: Failed building wheel for lxml -> see also https://github.com/flyte/upnpclient/pull/44/files
+    #   pip3 install lxml==5.3.1
+    #   pip3 install upnpclient --no-dependencies lxml
+    print_pip_info "lxml" "5.3.1"
     print_pip_info "upnpclient"
     print_pip_info "beautifulsoup4"
 
@@ -77,6 +84,8 @@ IL15_emulated_checks_init() {
 
       # EMBAbite fuzzer used this:
       # pip_install "upnpclient"
+      pip_install "lxml==5.3.1"
+      pip_install "upnpclient" "--no-dependencies lxml"
 
       ;;
     esac

modules/F17_cve_bin_tool.sh

@@ -41,6 +41,12 @@ F17_cve_bin_tool() {
   mkdir "${LOG_PATH_MODULE}/exploit/" || true
 
   print_output "[*] Loading SBOM ..." "no_log"
+  if ! [[ -f "${lEMBA_SBOM_JSON}" ]]; then
+    print_error "[-] No SBOM available!"
+    module_end_log "${FUNCNAME[0]}" "${lNEG_LOG}"
+    return
+  fi
+
   # read each item in the JSON array to an item in the Bash array
   readarray -t lSBOM_ARR < <(jq --compact-output '.components[]' "${lEMBA_SBOM_JSON}" || print_error "[-] SBOM loading error - Vulnerability analysis not available")
 

modules/P20_foscam_decryptor.sh

@@ -127,7 +127,7 @@ foscam_ubi_extractor() {
     print_output "[*] Checking nandsim kernel module"
     if ! lsmod | grep -q "^nandsim[[:space:]]"; then
       lsmod | grep "nandsim" || true
-      print_output "[-] WARNING: Nandsim kernel module not loaded - can't proceed"
+      print_output "[-] WARNING: Nandsim kernel module loading issue - trying to proceed"
       # return
       #   # we need to load nandsim with some parameters - unload it before
       #   modprobe -r nandsim
@@ -153,24 +153,34 @@ foscam_ubi_extractor() {
       print_output "[*] Mounting ${ORANGE}${lUBI_DEV}${NC} ubi device to ${ORANGE}${lUBI_MNT_PT}${NC}"
       mkdir -p "${lUBI_MNT_PT}" || true
       mount -t ubifs "${lUBI_DEV}" "${lUBI_MNT_PT}"
-      print_output "[*] Copy mounted ubi device to ${ORANGE}${lEXTRACTION_DIR_}/${lUBI_DEV}${NC}"
-      mkdir -p "${lEXTRACTION_DIR_}/${lUBI_DEV}"
-      cp -pri "${lUBI_MNT_PT}" "${lEXTRACTION_DIR_}/${lUBI_DEV}"
+      print_output "[*] Copy mounted ubi device to ${ORANGE}${lEXTRACTION_DIR_%\/}/${lUBI_DEV}${NC}"
+      mkdir -p "${lEXTRACTION_DIR_%\/}/${lUBI_DEV}"
+      cp -pri "${lUBI_MNT_PT}" "${lEXTRACTION_DIR_%\/}/${lUBI_DEV}"
+      # after this we should have a ubi image in our extraction directory. This should be extractable via unblob
+      print_output "[*] Umount ubi device from ${ORANGE}${lUBI_MNT_PT}/${lUBI_DEV}${NC}"
       umount "${lUBI_MNT_PT}" || true
       rm -r "${lUBI_MNT_PT}" || true
     done
 
     # do some cleanup
     print_output "[*] Detaching ubi device"
     ubidetach -d 0 || true
+    # ensure we have some extracted ubifs:
+    lUBI_FS_TARGET=$(find "${lEXTRACTION_DIR_%\/}/${lUBI_DEV}" -name ubifs)
+    if [[ -f "${lUBI_FS_TARGET}" ]]; then
+      # unblobber "${lUBI_FS_TARGET}" "${lEXTRACTION_DIR_%\/}_unblob_extracted" 1
+      binwalker_matryoshka "${lUBI_FS_TARGET}" "${lEXTRACTION_DIR_%\/}_binwalk_extracted"
+    fi
+
     # print_output "[*] Unloading nandsim module"
     # modprobe -r nandsim || true
     # print_output "[*] Unloading ubi module"
     # modprobe -r ubi || true
 
-    if [[ -d "${lEXTRACTION_DIR_}" ]]; then
-      lFOSCAM_UBI_FILES=$(find "${lEXTRACTION_DIR_}" -type f | wc -l)
-      lFOSCAM_UBI_DIRS=$(find "${lEXTRACTION_DIR_}" -type d | wc -l)
+    if [[ -d "${lEXTRACTION_DIR_%\/}_binwalk_extracted" ]]; then
+      print_output "[*] Checking ${lEXTRACTION_DIR_%\/}_binwalk_extracted for files and directories"
+      lFOSCAM_UBI_FILES=$(find "${lEXTRACTION_DIR_%\/}_binwalk_extracted" -type f | wc -l)
+      lFOSCAM_UBI_DIRS=$(find "${lEXTRACTION_DIR_%\/}_binwalk_extracted" -type d | wc -l)
     fi
 
     if [[ "${lFOSCAM_UBI_FILES}" -gt 0 ]]; then

modules/S06_distribution_identification.sh

@@ -59,7 +59,6 @@ S06_distribution_identification()
           continue
         fi
       fi
-      # mapfile -t lFOUND_FILES_ARR < <(find "${FIRMWARE_PATH}" -xdev -iwholename "*${lSEARCH_FILE}" || true)
       mapfile -t lFOUND_FILES_ARR < <(grep "${lSEARCH_FILE};" "${P99_CSV_LOG}" | cut -d ';' -f1 || true)
       for lFILE in "${lFOUND_FILES_ARR[@]}"; do
         # print_output "lFILE: ${lFILE}"
@@ -69,12 +68,14 @@ S06_distribution_identification()
           lSED_COMMAND="$(echo "${lCONFIG}" | cut -d\; -f4)"
           lFILE_QUOTED=$(escape_echo "${lFILE}")
           lOUT1="$(eval "${lPATTERN}" "${lFILE_QUOTED}" || true)"
+          lOUT1="${lOUT1//\'}"
+          # print_output "lCONFIG: ${lCONFIG}"
           # print_output "lPATTERN: ${lPATTERN}"
           # print_output "SED command: ${lSED_COMMAND}"
           # print_output "FILE: ${lFILE_QUOTED}"
           # print_output "identified before: ${lOUT1}"
           lOUT1=$(echo "${lOUT1}" | sort -u | tr '\n' ' ')
-          lOUT1=$(echo "${lOUT1}" | tr -d '"')
+          lOUT1="${lOUT1//\"}"
           # print_output "identified mod: ${lOUT1}"
           if [[ -n "${lSED_COMMAND}" ]]; then
             lIDENTIFIER=$(echo "${lOUT1}" | eval "${lSED_COMMAND}" | sed 's/  \+/ /g' | sed 's/ $//' || true)