Add address/undefined behaviour/thread sanitizer tests (#7334)

Adapt/re-use the pydebug test run to also add test with the various
clang sanitizers.

Address sanitizer and undefined behaviour sanitizer are run in parallel
to save a little time. Thread sanitizer is run individual for both
regular and freethreading builds.

At present I don't think thread sanitizer is hugely useful (since the
vast majority of tests run single-threaded in series). I have some plans
to improve that in future.

Address sanitizer will stop if it hits an error. The others will keep
going and write to some log files (and examination of those log files
triggers pass/failure)

Author: da-woods

.github/workflows/compiled_python.yml

@@ -0,0 +1,145 @@
+name: Run with compiled Python
+
+on:
+  workflow_call:
+    inputs:
+      sanitize:
+        required: false
+        default:
+        type: string
+      compiler:
+        required: false
+        default:
+        type: string
+      cpp_compiler:
+        required: false
+        default:
+        type: string
+      name:
+        required: true
+        type: string
+
+jobs:
+  do_run:
+    name: ${{inputs.name}}
+    runs-on: ubuntu-latest
+
+    env:
+      BACKEND: c,cpp
+      PYTHON_VERSION: 3.x-dev
+      CONFIGURE_ARGS: --with-pydebug
+      SANITIZER_CFLAGS: ""
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v5
+
+      - name: Set compiler
+        if: ${{inputs.compiler}}
+        run: |
+          CC=${{inputs.compiler}}
+          CXX=${{inputs.cpp_compiler}}
+          echo EXTERNAL_OVERRIDE_CC=1 >> $GITHUB_ENV
+          clangv=$($CC -v 2> >(grep "clang version"))
+          echo $clangv
+          if [[ $clangv == *"version 18"* && "${{inputs.sanitize}}" == *"thread"* ]]; then
+              # Python uses clang-17 instead of 18 because of bugs so do the same
+              CC=clang-17
+              CXX=clang-17
+          fi
+          echo "CC=$CC" >> $GITHUB_ENV
+          echo "CXX=$CXX" >> $GITHUB_ENV
+              
+
+      - name: Set up sanitizer args
+        if: ${{inputs.sanitize}}
+        run: |
+          SANITIZER_CFLAGS=""
+          CONFIGURE_ARGS=""
+          EXTRA_CONFIGURE_CFLAGS=""
+          EXCLUDE="run[.] memoryview[.] --no-examples"
+          if [[ "${{inputs.sanitize}}" == *"address"* ]]; then
+            CONFIGURE_ARGS="$CONFIGURE_ARGS --with-address-sanitizer --without-pymalloc"
+            SANITIZER_CFLAGS="$SANITIZER_CFLAGS -fsanitize=address"
+            echo "ASAN_OPTIONS=detect_leaks=false log_path=${{ github.workspace }}/san_log" >> $GITHUB_ENV
+          fi
+          # TODO - memory sanitizer requires rebuilding almost all of CPython's dependencies
+          # with memory sanitizer too, so isn't really usable for us.
+          if [[ "${{inputs.sanitize}}" == *"memory"* ]]; then
+            CONFIGURE_ARGS="$CONFIGURE_ARGS --with-memory-sanitizer"
+            SANITIZER_CFLAGS="$SANITIZER_CFLAGS -fsanitize=memory"
+          fi
+          if [[ "${{inputs.sanitize}}" == *"undefined"* ]]; then
+            CONFIGURE_ARGS="$CONFIGURE_ARGS --with-undefined-behavior-sanitizer"
+            # We call functions through slightly incorrect pointer types a lot so disable this check for now for now
+            EXTRA_CONFIGURE_CFLAGS="$EXTRA_CONFIGURE_CFLAGS -fno-sanitize=function"
+            # omit vptr because it's largely C++-only and requires linking with clang++ (which breaks other things)
+            SANITIZER_CFLAGS="$SANITIZER_CFLAGS -fsanitize=undefined -fno-sanitize=function -fno-sanitize=vptr -fno-omit-frame-pointer"
+            echo "print_stacktrace=1" >> $GITHUB_ENV
+            EXCLUDE="$EXCLUDE --excludefile tests/ubsan_bugs.txt"
+            echo "UBSAN_OPTIONS=log_path=${{ github.workspace }}/san_log" >> $GITHUB_ENV
+          fi
+          if [[ "${{inputs.sanitize}}" == *"thread"* ]]; then
+            CONFIGURE_ARGS="$CONFIGURE_ARGS --with-thread-sanitizer"
+            SANITIZER_CFLAGS="$SANITIZER_CFLAGS -fsanitize=thread"
+            if [[ "${{inputs.sanitize}}" == *"ft"* ]]; then
+              echo "PYTHON_VERSION=3.xt-dev" >> $GITHUB_ENV
+              CONFIGURE_ARGS="$CONFIGURE_ARGS --disable-gil"
+              TSAN_SUPPRESSIONS="${GITHUB_WORKSPACE}/cpython_main/Tools/tsan/suppressions_free_threading.txt"
+            else
+              TSAN_SUPPRESSIONS="${GITHUB_WORKSPACE}/cpython_main/Tools/tsan/suppressions.txt"
+            fi
+            if [[ "${{github.event.label.name}}" != "full_sanitizers" ]]; then
+              # For thread sanitizer run on a much smaller list of tests
+              EXCLUDE="tag:threads"
+            fi
+            EXCLUDE="$EXCLUDE --excludefile tests/tsan_bugs.txt"
+            echo "TSAN_OPTIONS=suppressions=$TSAN_SUPPRESSIONS log_path=${{ github.workspace }}/san_log" >> $GITHUB_ENV
+            # Having too many workers seems to lead to an exit without a diagnostic message - possibly memory?
+            echo "TEST_PARALLELISM=-j3" >> $GITHUB_ENV
+          fi
+          # Even running Python like this is slow, so only run a subset of tests
+          # (i.e. compile-only tests tell us nothing)
+          echo "EXCLUDE=$EXCLUDE --no-refnanny" >> $GITHUB_ENV
+          # https://github.com/google/sanitizers/issues/934
+          echo "LD_PRELOAD=$(realpath "$(clang -print-file-name=libstdc++.so)")" >> $GITHUB_ENV
+          echo "CONFIGURE_ARGS=$CONFIGURE_ARGS" >> $GITHUB_ENV
+          echo "SANITIZER_CFLAGS=$SANITIZER_CFLAGS" >> $GITHUB_ENV
+          echo "EXTRA_CONFIGURE_CFLAGS=$EXTRA_CONFIGURE_CFLAGS" >> $GITHUB_ENV 
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update -y -q
+          sudo apt-get install -y -q libbz2-dev lzma-dev libreadline-dev libgmp-dev
+
+      - name: Build Python
+        run: |
+          git clone https://github.com/python/cpython/ cpython_main
+          cd cpython_main
+          ./configure ${CONFIGURE_ARGS} --prefix=${GITHUB_WORKSPACE}/cpython_install CFLAGS="-O2 $EXTRA_CONFIGURE_CFLAGS"
+          make -j8
+          make install
+          ${GITHUB_WORKSPACE}/cpython_install/bin/python3 -m venv ${GITHUB_WORKSPACE}/venv_pydebug
+
+      - name: Run CI
+        run: |
+          cd "${GITHUB_WORKSPACE}/"
+          source venv_pydebug/bin/activate
+          bash ./Tools/ci-run.sh
+
+      - name: Archive logs
+        if: ${{ inputs.sanitize && always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{inputs.sanitize}}-logs
+          path: san_log.*
+          if-no-files-found: ignore
+
+      # The check of the sanitizer logs does a bit of filtering of unwanted diagnostics
+      # and finishes with an error code if anything bad is found.
+      - name: Check logs
+        if:  ${{ inputs.sanitize && always() }}
+        run: |
+          cd "${GITHUB_WORKSPACE}/"
+          source venv_pydebug/bin/activate
+          python Tools/examine_sanitizer_logs.py san_log.*

Cython/Utility/Exceptions.c

@@ -18,6 +18,11 @@ if (likely(__Pyx_init_assertions_enabled() == 0)); else
   static int __pyx_assertions_enabled_flag;
   #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag)
 
+  #if __clang__ || __GNUC__
+  // "Assertions enabled" may be written multiple times when using subinterpreters.
+  // However, it should always be written to the same value to isn't a "real" race.
+  __attribute__((no_sanitize("thread")))
+  #endif
   static int __Pyx_init_assertions_enabled(void) {
     PyObject *builtins, *debug, *debug_str;
     int flag;

Cython/Utility/Optimize.c

@@ -1191,6 +1191,13 @@ static {{c_ret_type}} __Pyx_Fallback_{{cfunc_name}}(PyObject *op1, PyObject *op2
 }
 
 #if CYTHON_USE_PYLONG_INTERNALS
+{{if op == 'Lshift'}}
+#if __clang__ || __GNUC__
+// left-shift by more than the width of the number is undefined behaviour.
+// We do check it (and test that it gives the right answer though).
+__attribute__((no_sanitize("shift")))
+#endif
+{{endif}}
 static {{c_ret_type}} __Pyx_Unpacked_{{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) {
     CYTHON_MAYBE_UNUSED_VAR(inplace);
     CYTHON_UNUSED_VAR(zerodivision_check);

Tools/ci-run.sh

@@ -7,7 +7,7 @@ GCC_VERSION=${GCC_VERSION:=10}
 # Set up compilers
 if [[ $TEST_CODE_STYLE == "1" ]]; then
   echo "Skipping compiler setup: Code style run"
-elif [[ $OSTYPE == "linux-gnu"* ]]; then
+elif [[ $OSTYPE == "linux-gnu"* && ! "$EXTERNAL_OVERRIDE_CC" ]]; then
   echo "Setting up linux compiler"
   echo "Installing requirements [apt]"
   sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test"
@@ -102,7 +102,7 @@ if [[ $PYTHON_VERSION != *"t" && $PYTHON_VERSION != *"t-dev" ]]; then
 fi
 if [[ $PYTHON_VERSION != *"-dev" ]]; then
   python -m pip install --pre -r test-requirements.txt || exit 1
-else
+elif [[ ! "$SANITIZER_CFLAGS" ]]; then
   # Install packages one by one, allowing failures due to missing recent wheels.
   cat test-requirements.txt | while read package; do python -m pip install --pre --only-binary ":all:" "$package" || true; done
 fi
@@ -146,7 +146,7 @@ if [[ $OSTYPE == "msys" ]]; then  # for MSVC cl
   # (off by default) 5045 warns that the compiler will insert Spectre mitigations for memory load if the /Qspectre switch is specified
   # (off by default) 4820 warns about the code in Python\3.9.6\x64\include ...
   CFLAGS="-Od /Z7 /MP /W4 /wd4711 /wd4127 /wd5045 /wd4820"
-elif [[ $OSTYPE == "darwin"* ]]; then
+elif [[ $OSTYPE == "darwin"* || $CC == "clang" ]]; then
   CFLAGS="-O0 -g2 -Wall -Wextra -Wcast-qual -Wconversion -Wdeprecated -Wunused-result"
 else
   CFLAGS="-Og -g2 -Wall -Wextra -Wcast-qual -Wconversion -Wdeprecated -Wunused-result"
@@ -162,6 +162,10 @@ elif [[ $ODD_VERSION == "0" ]]; then
     CFLAGS="$CFLAGS -UNDEBUG"
 fi
 
+if [[ "$SANITIZER_CFLAGS" ]]; then
+    CFLAGS="$CFLAGS $SANITIZER_CFLAGS"
+fi
+
 if [[ $NO_CYTHON_COMPILE != "1" && $PYTHON_VERSION != "pypy"* ]]; then
 
   BUILD_CFLAGS="$CFLAGS -O2"
@@ -229,9 +233,13 @@ if [[ $COVERAGE == "1" ]]; then
   RUNTESTS_ARGS="$RUNTESTS_ARGS --coverage --coverage-html --coverage-md --cython-only"
 fi
 if [[ $TEST_CODE_STYLE != "1" ]]; then
-  RUNTESTS_ARGS="$RUNTESTS_ARGS -j7"
+  if [[ ! $TEST_PARALLELISM ]]; then
+    TEST_PARALLELISM=-j7
+  fi
+  RUNTESTS_ARGS="$RUNTESTS_ARGS $TEST_PARALLELISM"
 fi
 
+
 if [[ $PYTHON_VERSION == "graalpy"* ]]; then
   # [DW] - the Graal JIT and Cython don't seem to get on too well. Disabling the
   # JIT actually makes it faster! And reduces the number of cores each process uses.

Tools/examine_sanitizer_logs.py

@@ -0,0 +1,33 @@
+import sys
+import re
+
+POS_MATCH = re.compile(r"^[^:]+:\d+:\d+: ")
+
+def check_file(filename) -> int:
+    failed_count = 0
+    with open(filename) as f:
+        for line in f:
+            if not POS_MATCH.match(line) or line.startswith("WARNING: ThreadSanitizer"):
+                continue
+            if line.startswith("conftest.c"):
+                # this is in the Python setup - we don't care
+                continue
+            if "applying zero offset to null pointer" in line:
+                # This is OK in C++ and dropped in clang21 (on DW's laptop) so treat it as fine.
+                continue
+            # anything not specifically included is a failure
+            failed_count += 1
+            print(line)
+    return failed_count
+
+failed_count = 0
+
+if len(sys.argv) == 2 and sys.argv[1].endswith('.*'):
+    # No issues so the pattern has not expanded
+    print(f"No logs found with pattern '{sys.argv[1]}'")
+    exit(0)
+for arg in sys.argv[1:]:
+    print(f"Looking at file '{arg}':")
+    failed_count += check_file(arg)
+
+exit(failed_count)

runtests.py

@@ -2150,7 +2150,7 @@ def test_embed(self):
 
 def load_listfile(filename):
     # just reuse the FileListExclude implementation
-    return list(FileListExcluder(filename))
+    return FileListExcluder(filename)
 
 class MissingDependencyExcluder(object):
     def __init__(self, deps):
@@ -2451,6 +2451,10 @@ def main():
         "--listfile", dest="listfile",
         action="append",
         help="specify a file containing a list of tests to run")
+    parser.add_argument(
+        "--excludefile", dest="excludefile",
+        action="append",
+        help="specify a file containing a list of tests to run")
     parser.add_argument(
         "-j", "--shard_count", dest="shard_count", metavar="N",
         type=int, default=1,
@@ -2590,7 +2594,7 @@ def main():
 
     if options.listfile:
         for listfile in options.listfile:
-            cmd_args.extend(load_listfile(listfile))
+            cmd_args.extend(load_listfile(listfile).excludes.keys())
 
     if options.capture and not options.for_debugging:
         keep_alive_interval = 10
@@ -2914,6 +2918,10 @@ def runtests(options, cmd_args, coverage=None):
     if options.exclude:
         exclude_selectors += [ string_selector(r) for r in options.exclude ]
 
+    if options.excludefile:
+        for excludefile in options.excludefile:
+            exclude_selectors.append(load_listfile(excludefile))
+
     if not COMPILER_HAS_INT128:
         exclude_selectors += [RegExSelector('int128')]