WIP

ntop · Jan 9, 2025 · 82d2ef0 · 82d2ef0
1 parent 89ccc30
commit 82d2ef0
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 15 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -130,13 +130,6 @@ jobs:
             msan: ""
             nBPF: ""
             global_context: "--disable-global-context-support"
-          - compiler: "clang" # TODO: some issues with masan/clang/ubuntu-24.04
-            os: ubuntu-22.04
-            gcrypt: ""
-            pcre: "--with-pcre2"
-            maxminddb: "--with-maxminddb"
-            msan: "--with-memory-sanitizer --disable-memory-track-origins"
-            nBPF: ""
           - compiler: "cc"
             os: macOS-13
             gcrypt: ""
@@ -251,8 +244,7 @@ jobs:
         run: |
           NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
       - name: Test nDPI [UNIT]
-        # Some issues with masan + json-c. Disable the test as workaround
-        if: ${{ !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') }}
+        if: ${{ !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') }}
         run: |
           ./tests/do-unit.sh
       - name: Test nDPI [DGA]

diff --git a/.github/workflows/build_masan.yml b/.github/workflows/build_masan.yml
@@ -0,0 +1,47 @@
+name: Memory Sanitizer builds
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+    branches:
+      - dev
+    types: [opened, synchronize, reopened]
+  release:
+    types: [created]
+  workflow_dispatch:
+  schedule:
+    #At the end of every day
+    - cron: '0 0 * * *'
+
+jobs:
+  memorysanitizer:
+    name: Memory Sanitizer (ubuntu-22.04)
+    runs-on: ubuntu-22.04 # TODO: some issues with masan/clang/ubuntu-24.04
+    env:
+      CC: clang
+      CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS
+      # Idea to have significant faster tests while pushing/merging: test all pcaps only on schedule runs; otherwise tests only the pcaps updated/added recently
+      NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS: ${{ (github.event_name == 'schedule') && '0' || '1' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install Ubuntu Prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c-dev libpcap-dev rrdtool librrd-dev parallel libpcre3-dev libmaxminddb-dev
+      - name: Configure nDPI
+        run: |
+          ./autogen.sh --enable-option-checking=fatal --with-pcre2 --with-maxminddb --with-memory-sanitizer --disable-memory-track-origins
+      - name: Build nDPI
+        run: |
+          make -j $(nproc) all
+          make -j $(nproc) -C example ndpiSimpleIntegration
+          make -j $(nproc) -C rrdtool
+      - name: Tests
+        run: |
+          NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
+          # Some issues with masan + json-c. Disable the test as workaround
+          # ./tests/do-unit.sh
+          ./tests/do-dga.sh
diff --git a/.github/workflows/build_no_x86_64.yml b/.github/workflows/build_no_x86_64.yml
@@ -9,13 +9,18 @@ on:
     types: [opened, synchronize, reopened]
   release:
     types: [created]
+  schedule:
+    # At the end of every day
+    - cron: '0 0 * * *'
 jobs:
 
   test:
     name: Ubuntu ${{ matrix.arch }}
     runs-on: ubuntu-latest
     env:
       CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS
+      # Idea to have significant faster tests while pushing/merging: test all pcaps only on schedule runs; otherwise tests only the pcaps updated/added recently
+      NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS: ${{ (github.event_name == 'schedule') && '0' || '1' }}
     strategy:
       fail-fast: true
       matrix:
@@ -45,7 +50,7 @@ jobs:
             "apt-get -y update &&
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
-            env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
+            env CC=gcc ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
             make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
@@ -69,7 +74,7 @@ jobs:
             "apt-get -y update &&
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
-            env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
+            env CC=gcc ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
             make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
@@ -93,7 +98,7 @@ jobs:
             "apt-get -y update &&
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
-            env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
+            env CC=gcc ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
             make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&

diff --git a/tests/do.sh.in b/tests/do.sh.in
@@ -31,6 +31,11 @@ if [ "${NDPI_SKIP_PARALLEL_BAR}" = "1" ]; then
    SKIP_PARALLEL_BAR=1
 fi
 
+TEST_ONLY_RECENTLY_UPDATED_PCAPS=0
+if [ "${NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS}" = "1" ]; then
+   TEST_ONLY_RECENTLY_UPDATED_PCAPS=1
+fi
+
 #Remember: valgrind and *SAN are incompatible!
 CMD_PREFIX="${CMD_PREFIX}"
 if [ "${NDPI_TESTS_WINE}" = "1" ]; then
@@ -239,10 +244,16 @@ for d in $(find ./cfgs/* -type d -maxdepth 0 2>/dev/null) ; do
 
     cd ./cfgs/"$(basename "$d")" || exit 1
 
-    if [ "$#" -ne 0 ]; then
-        PCAPS=$*
+    if [ $TEST_ONLY_RECENTLY_UPDATED_PCAPS -eq 1 ]; then
+        # Latest 10 pcaps updatetd recently (to be more precise, pcap which triggered a change
+        # in their results lately)
+        PCAPS=$(cd result || exit 1; find * -type f -print0  | xargs -0 ls -t | head -10 | xargs basename -s .out)
     else
-        PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*)
+        if [ "$#" -ne 0 ]; then
+            PCAPS=$*
+        else
+            PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*)
+        fi
     fi
     FAILURES=()
     READER_EXTRA_PARAM=""