Skip to content

Commit

Permalink
[3.10] bpo-46633: Skip tests on ASAN and/or MSAN builds (pythonGH-31632
Browse files Browse the repository at this point in the history
…) (pythonGH-31634)

* Refactor sanitiser skip tests into test.support (pythonGH-30889)

* Refactor sanitizer skip tests into test.support

(cherry picked from commit b1cb843)

* Add skips to crashing tests under sanitizers instead of manually skipping them (pythonGH-30897)

(cherry picked from commit a275053)

* bpo-46633: Skip tests on ASAN and/or MSAN builds (pythonGH-31632)

Skip tests on ASAN and/or MSAN builds:

* multiprocessing tests
* test___all__
* test_concurrent_futures
* test_decimal
* test_peg_generator
* test_tools

(cherry picked from commit 9204bb7)

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
  • Loading branch information
vstinner and pablogsal authored Mar 2, 2022
1 parent 7dbb2f8 commit 9326445
Show file tree
Hide file tree
Showing 14 changed files with 109 additions and 49 deletions.
6 changes: 6 additions & 0 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@
msvcrt = None


if support.check_sanitizer(address=True):
# bpo-45200: Skip multiprocessing tests if Python is built with ASAN to
# work around a libasan race condition: dead lock in pthread_create().
raise unittest.SkipTest("libasan has a pthread_create() dead lock")


def latin(s):
return s.encode('latin')

Expand Down
37 changes: 36 additions & 1 deletion Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
"requires_IEEE_754", "requires_zlib",
"anticipate_failure", "load_package_tests", "detect_api_mismatch",
"check__all__", "skip_if_buggy_ucrt_strfptime",
"check_disallow_instantiation",
"check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer",
# sys
"is_jython", "is_android", "check_impl_detail", "unix_shell",
"setswitchinterval",
Expand Down Expand Up @@ -367,6 +367,41 @@ def wrapper(*args, **kw):
return decorator


def check_sanitizer(*, address=False, memory=False, ub=False):
"""Returns True if Python is compiled with sanitizer support"""
if not (address or memory or ub):
raise ValueError('At least one of address, memory, or ub must be True')


_cflags = sysconfig.get_config_var('CFLAGS') or ''
_config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
memory_sanitizer = (
'-fsanitize=memory' in _cflags or
'--with-memory-sanitizer' in _config_args
)
address_sanitizer = (
'-fsanitize=address' in _cflags or
'--with-memory-sanitizer' in _config_args
)
ub_sanitizer = (
'-fsanitize=undefined' in _cflags or
'--with-undefined-behavior-sanitizer' in _config_args
)
return (
(memory and memory_sanitizer) or
(address and address_sanitizer) or
(ub and ub_sanitizer)
)


def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False):
"""Decorator raising SkipTest if running with a sanitizer active."""
if not reason:
reason = 'not working with sanitizers active'
skip = check_sanitizer(address=address, memory=memory, ub=ub)
return unittest.skipIf(skip, reason)


def system_must_validate_cert(f):
"""Skip the test on TLS certificate validation failures."""
@functools.wraps(f)
Expand Down
7 changes: 7 additions & 0 deletions Lib/test/test___all__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@
import sys


if support.check_sanitizer(address=True, memory=True):
# bpo-46633: test___all__ is skipped because importing some modules
# directly can trigger known problems with ASAN (like tk or crypt).
raise unittest.SkipTest("workaround ASAN build issues on loading tests "
"like tk or crypt")


class NoAll(RuntimeError):
pass

Expand Down
6 changes: 6 additions & 0 deletions Lib/test/test_concurrent_futures.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@
import multiprocessing.util


if support.check_sanitizer(address=True, memory=True):
# bpo-46633: Skip the test because it is too slow when Python is built
# with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions.
raise unittest.SkipTest("test too slow on ASAN/MSAN build")


def create_future(state=PENDING, exception=None, result=None):
f = Future()
f._state = state
Expand Down
3 changes: 3 additions & 0 deletions Lib/test/test_crypt.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import sys
import unittest
from test.support import check_sanitizer


try:
if check_sanitizer(address=True, memory=True):
raise unittest.SkipTest("The crypt module SEGFAULTs on ASAN/MSAN builds")
import crypt
IMPORT_ERROR = None
except ImportError as ex:
Expand Down
16 changes: 3 additions & 13 deletions Lib/test/test_decimal.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import locale
from test.support import (run_unittest, run_doctest, is_resource_enabled,
requires_IEEE_754, requires_docstrings,
requires_legacy_unicode_capi)
requires_legacy_unicode_capi, check_sanitizer)
from test.support import (TestFailed,
run_with_locale, cpython_only,
darwin_malloc_err_warning)
Expand All @@ -43,17 +43,6 @@
import random
import inspect
import threading
import sysconfig
_cflags = sysconfig.get_config_var('CFLAGS') or ''
_config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
MEMORY_SANITIZER = (
'-fsanitize=memory' in _cflags or
'--with-memory-sanitizer' in _config_args
)

ADDRESS_SANITIZER = (
'-fsanitize=address' in _cflags
)


if sys.platform == 'darwin':
Expand Down Expand Up @@ -5511,7 +5500,8 @@ def __abs__(self):
# Issue 41540:
@unittest.skipIf(sys.platform.startswith("aix"),
"AIX: default ulimit: test is flaky because of extreme over-allocation")
@unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing "
@unittest.skipIf(check_sanitizer(address=True, memory=True),
"ASAN/MSAN sanitizer defaults to crashing "
"instead of returning NULL for malloc failure.")
def test_maxcontext_exact_arith(self):

Expand Down
20 changes: 5 additions & 15 deletions Lib/test/test_faulthandler.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
import signal
import subprocess
import sys
import sysconfig
from test import support
from test.support import os_helper
from test.support import script_helper, is_android
from test.support import skip_if_sanitizer
import tempfile
import unittest
from textwrap import dedent
Expand All @@ -21,16 +21,6 @@

TIMEOUT = 0.5
MS_WINDOWS = (os.name == 'nt')
_cflags = sysconfig.get_config_var('CFLAGS') or ''
_config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
UB_SANITIZER = (
'-fsanitize=undefined' in _cflags or
'--with-undefined-behavior-sanitizer' in _config_args
)
MEMORY_SANITIZER = (
'-fsanitize=memory' in _cflags or
'--with-memory-sanitizer' in _config_args
)


def expected_traceback(lineno1, lineno2, header, min_count=1):
Expand Down Expand Up @@ -310,8 +300,8 @@ def test_gil_released(self):
3,
'Segmentation fault')

@unittest.skipIf(UB_SANITIZER or MEMORY_SANITIZER,
"sanitizer builds change crashing process output.")
@skip_if_sanitizer(memory=True, ub=True, reason="sanitizer "
"builds change crashing process output.")
@skip_segfault_on_android
def test_enable_file(self):
with temporary_filename() as filename:
Expand All @@ -327,8 +317,8 @@ def test_enable_file(self):

@unittest.skipIf(sys.platform == "win32",
"subprocess doesn't support pass_fds on Windows")
@unittest.skipIf(UB_SANITIZER or MEMORY_SANITIZER,
"sanitizer builds change crashing process output.")
@skip_if_sanitizer(memory=True, ub=True, reason="sanitizer "
"builds change crashing process output.")
@skip_segfault_on_android
def test_enable_fd(self):
with tempfile.TemporaryFile('wb+') as fp:
Expand Down
4 changes: 4 additions & 0 deletions Lib/test/test_idle.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import unittest
from test.support.import_helper import import_module
from test.support import check_sanitizer

if check_sanitizer(address=True, memory=True):
raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds")

# Skip test_idle if _tkinter wasn't built, if tkinter is missing,
# if tcl/tk is not the 8.5+ needed for ttk widgets,
Expand Down
25 changes: 7 additions & 18 deletions Lib/test/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import random
import signal
import sys
import sysconfig
import textwrap
import threading
import time
Expand All @@ -44,6 +43,7 @@
from test.support import os_helper
from test.support import threading_helper
from test.support import warnings_helper
from test.support import skip_if_sanitizer
from test.support.os_helper import FakePath

import codecs
Expand All @@ -66,17 +66,6 @@ def byteslike(*pos, **kw):
class EmptyStruct(ctypes.Structure):
pass

_cflags = sysconfig.get_config_var('CFLAGS') or ''
_config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
MEMORY_SANITIZER = (
'-fsanitize=memory' in _cflags or
'--with-memory-sanitizer' in _config_args
)

ADDRESS_SANITIZER = (
'-fsanitize=address' in _cflags
)

# Does io.IOBase finalizer log the exception if the close() method fails?
# The exception is ignored silently by default in release build.
IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)
Expand Down Expand Up @@ -1550,8 +1539,8 @@ def test_truncate_on_read_only(self):
class CBufferedReaderTest(BufferedReaderTest, SizeofTest):
tp = io.BufferedReader

@unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing "
"instead of returning NULL for malloc failure.")
@skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing "
"instead of returning NULL for malloc failure.")
def test_constructor(self):
BufferedReaderTest.test_constructor(self)
# The allocation can succeed on 32-bit builds, e.g. with more
Expand Down Expand Up @@ -1915,8 +1904,8 @@ def test_slow_close_from_thread(self):
class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
tp = io.BufferedWriter

@unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing "
"instead of returning NULL for malloc failure.")
@skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing "
"instead of returning NULL for malloc failure.")
def test_constructor(self):
BufferedWriterTest.test_constructor(self)
# The allocation can succeed on 32-bit builds, e.g. with more
Expand Down Expand Up @@ -2414,8 +2403,8 @@ def test_interleaved_readline_write(self):
class CBufferedRandomTest(BufferedRandomTest, SizeofTest):
tp = io.BufferedRandom

@unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing "
"instead of returning NULL for malloc failure.")
@skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing "
"instead of returning NULL for malloc failure.")
def test_constructor(self):
BufferedRandomTest.test_constructor(self)
# The allocation can succeed on 32-bit builds, e.g. with more
Expand Down
12 changes: 10 additions & 2 deletions Lib/test/test_peg_generator/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
import os

import os.path
import unittest
from test import support
from test.support import load_package_tests


if support.check_sanitizer(address=True, memory=True):
# bpo-46633: Skip the test because it is too slow when Python is built
# with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions.
raise unittest.SkipTest("test too slow on ASAN/MSAN build")


# Load all tests in package
def load_tests(*args):
return load_package_tests(os.path.dirname(__file__), *args)
5 changes: 5 additions & 0 deletions Lib/test/test_tix.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@
import unittest
from test import support
from test.support import import_helper
from test.support import check_sanitizer

if check_sanitizer(address=True, memory=True):
raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds")


# Skip this test if the _tkinter module wasn't built.
_tkinter = import_helper.import_module('_tkinter')
Expand Down
6 changes: 6 additions & 0 deletions Lib/test/test_tk.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import unittest
from test import support
from test.support import import_helper
from test.support import check_sanitizer

if check_sanitizer(address=True, memory=True):
raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds")

# Skip test if _tkinter wasn't built.
import_helper.import_module('_tkinter')

Expand Down
7 changes: 7 additions & 0 deletions Lib/test/test_tools/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@
from test import support
from test.support import import_helper


if support.check_sanitizer(address=True, memory=True):
# bpo-46633: Skip the test because it is too slow when Python is built
# with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions.
raise unittest.SkipTest("test too slow on ASAN/MSAN build")


basepath = os.path.normpath(
os.path.dirname( # <src/install dir>
os.path.dirname( # Lib
Expand Down
4 changes: 4 additions & 0 deletions Lib/test/test_ttk_guionly.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import unittest
from test import support
from test.support import import_helper
from test.support import check_sanitizer

if check_sanitizer(address=True, memory=True):
raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds")

# Skip this test if _tkinter wasn't built.
import_helper.import_module('_tkinter')
Expand Down

0 comments on commit 9326445

Please sign in to comment.