bpo-40280: Address more test failures on Emscripten (GH-31050)

Co-authored-by: Brett Cannon <brett@python.org>

Author: tiran

Lib/test/support/__init__.py

@@ -1278,6 +1278,8 @@ def reap_children():
     # Need os.waitpid(-1, os.WNOHANG): Windows is not supported
     if not (hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG')):
         return
+    elif not has_subprocess_support:
+        return
 
     # Reap all our dead child processes so we don't leave zombies around.
     # These hog resources and might be causing some of the buildbots to die.

Lib/test/support/os_helper.py

@@ -502,7 +502,7 @@ def __fspath__(self):
 def fd_count():
     """Count the number of open file descriptors.
     """
-    if sys.platform.startswith(('linux', 'freebsd')):
+    if sys.platform.startswith(('linux', 'freebsd', 'emscripten')):
         try:
             names = os.listdir("/proc/self/fd")
             # Subtract one because listdir() internally opens a file

Lib/test/test_builtin.py

@@ -393,6 +393,7 @@ def test_compile_top_level_await_no_coro(self):
                                 msg=f"source={source} mode={mode}")
 
 
+    @unittest.skipIf(support.is_emscripten, "socket.accept is broken")
     def test_compile_top_level_await(self):
         """Test whether code some top level await can be compiled.
 
@@ -1213,6 +1214,7 @@ def test_open_default_encoding(self):
             os.environ.clear()
             os.environ.update(old_environ)
 
+    @support.requires_subprocess()
     def test_open_non_inheritable(self):
         fileobj = open(__file__, encoding="utf-8")
         with fileobj:

Lib/test/test_capi.py

@@ -611,6 +611,7 @@ def check_fatal_error(self, code, expected, not_expected=()):
             self.assertNotIn(name, modules)
         self.assertEqual(len(modules), total)
 
+    @support.requires_subprocess()
     def test_fatal_error(self):
         # By default, stdlib extension modules are ignored,
         # but not test modules.
@@ -880,6 +881,7 @@ class Test_testinternalcapi(unittest.TestCase):
                     if name.startswith('test_'))
 
 
+@support.requires_subprocess()
 class PyMemDebugTests(unittest.TestCase):
     PYTHONMALLOC = 'debug'
     # '0x04c06e0' or '04C06E0'

Lib/test/test_faulthandler.py

@@ -19,6 +19,9 @@
 except ImportError:
     _testcapi = None
 
+if not support.has_subprocess_support:
+    raise unittest.SkipTest("test module requires subprocess")
+
 TIMEOUT = 0.5
 MS_WINDOWS = (os.name == 'nt')
 

Lib/test/test_fileio.py

@@ -9,7 +9,7 @@
 from weakref import proxy
 from functools import wraps
 
-from test.support import cpython_only, swap_attr, gc_collect
+from test.support import cpython_only, swap_attr, gc_collect, is_emscripten
 from test.support.os_helper import (TESTFN, TESTFN_UNICODE, make_bad_fd)
 from test.support.warnings_helper import check_warnings
 from collections import UserList
@@ -373,7 +373,7 @@ def testAbles(self):
             self.assertEqual(f.isatty(), False)
             f.close()
 
-            if sys.platform != "win32":
+            if sys.platform != "win32" and not is_emscripten:
                 try:
                     f = self.FileIO("/dev/tty", "a")
                 except OSError:

Lib/test/test_genericalias.py

@@ -24,14 +24,20 @@
 from fileinput import FileInput
 from itertools import chain
 from http.cookies import Morsel
-from multiprocessing.managers import ValueProxy
-from multiprocessing.pool import ApplyResult
+try:
+    from multiprocessing.managers import ValueProxy
+    from multiprocessing.pool import ApplyResult
+    from multiprocessing.queues import SimpleQueue as MPSimpleQueue
+except ImportError:
+    # _multiprocessing module is optional
+    ValueProxy = None
+    ApplyResult = None
+    MPSimpleQueue = None
 try:
     from multiprocessing.shared_memory import ShareableList
 except ImportError:
     # multiprocessing.shared_memory is not available on e.g. Android
     ShareableList = None
-from multiprocessing.queues import SimpleQueue as MPSimpleQueue
 from os import DirEntry
 from re import Pattern, Match
 from types import GenericAlias, MappingProxyType, AsyncGeneratorType
@@ -79,13 +85,14 @@ class BaseTest(unittest.TestCase):
                      Queue, SimpleQueue,
                      _AssertRaisesContext,
                      SplitResult, ParseResult,
-                     ValueProxy, ApplyResult,
                      WeakSet, ReferenceType, ref,
-                     ShareableList, MPSimpleQueue,
+                     ShareableList,
                      Future, _WorkItem,
                      Morsel]
     if ctypes is not None:
         generic_types.extend((ctypes.Array, ctypes.LibraryLoader))
+    if ValueProxy is not None:
+        generic_types.extend((ValueProxy, ApplyResult, MPSimpleQueue))
 
     def test_subscriptable(self):
         for t in self.generic_types:

Lib/test/test_getpass.py

@@ -28,6 +28,9 @@ def test_username_priorities_of_env_values(self, environ):
             getpass.getuser()
         except ImportError: # in case there's no pwd module
             pass
+        except KeyError:
+            # current user has no pwd entry
+            pass
         self.assertEqual(
             environ.get.call_args_list,
             [mock.call(x) for x in ('LOGNAME', 'USER', 'LNAME', 'USERNAME')])

Lib/test/test_inspect.py

@@ -788,6 +788,7 @@ def test_nested_class_definition_inside_function(self):
         self.assertSourceEqual(mod2.cls213, 218, 222)
         self.assertSourceEqual(mod2.cls213().func219(), 220, 221)
 
+    @unittest.skipIf(support.is_emscripten, "socket.accept is broken")
     def test_nested_class_definition_inside_async_function(self):
         import asyncio
         self.addCleanup(asyncio.set_event_loop_policy, None)

Lib/test/test_interpreters.py

@@ -5,7 +5,8 @@
 import unittest
 import time
 
-import _xxsubinterpreters as _interpreters
+from test.support import import_helper
+_interpreters = import_helper.import_module('_xxsubinterpreters')
 from test.support import interpreters
 
 

Lib/test/test_io.py

@@ -76,6 +76,10 @@ def _default_chunk_size():
     with open(__file__, "r", encoding="latin-1") as f:
         return f._CHUNK_SIZE
 
+requires_alarm = unittest.skipUnless(
+    hasattr(signal, "alarm"), "test requires signal.alarm()"
+)
+
 
 class MockRawIOWithoutRead:
     """A RawIO implementation without read(), so as to exercise the default
@@ -4435,12 +4439,15 @@ def _read():
                 if e.errno != errno.EBADF:
                     raise
 
+    @requires_alarm
     def test_interrupted_write_unbuffered(self):
         self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0)
 
+    @requires_alarm
     def test_interrupted_write_buffered(self):
         self.check_interrupted_write(b"xy", b"xy", mode="wb")
 
+    @requires_alarm
     def test_interrupted_write_text(self):
         self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii")
 
@@ -4472,9 +4479,11 @@ def on_alarm(*args):
             wio.close()
             os.close(r)
 
+    @requires_alarm
     def test_reentrant_write_buffered(self):
         self.check_reentrant_write(b"xy", mode="wb")
 
+    @requires_alarm
     def test_reentrant_write_text(self):
         self.check_reentrant_write("xy", mode="w", encoding="ascii")
 
@@ -4502,10 +4511,12 @@ def alarm_handler(sig, frame):
             os.close(w)
             os.close(r)
 
+    @requires_alarm
     def test_interrupted_read_retry_buffered(self):
         self.check_interrupted_read_retry(lambda x: x.decode('latin1'),
                                           mode="rb")
 
+    @requires_alarm
     def test_interrupted_read_retry_text(self):
         self.check_interrupted_read_retry(lambda x: x,
                                           mode="r", encoding="latin1")
@@ -4578,9 +4589,11 @@ def alarm2(sig, frame):
                 if e.errno != errno.EBADF:
                     raise
 
+    @requires_alarm
     def test_interrupted_write_retry_buffered(self):
         self.check_interrupted_write_retry(b"x", mode="wb")
 
+    @requires_alarm
     def test_interrupted_write_retry_text(self):
         self.check_interrupted_write_retry("x", mode="w", encoding="latin1")
 

Lib/test/test_os.py

@@ -992,6 +992,7 @@ def _empty_mapping(self):
     @unittest.skipUnless(unix_shell and os.path.exists(unix_shell),
                          'requires a shell')
     @unittest.skipUnless(hasattr(os, 'popen'), "needs os.popen()")
+    @support.requires_subprocess()
     def test_update2(self):
         os.environ.clear()
         os.environ.update(HELLO="World")
@@ -1002,6 +1003,7 @@ def test_update2(self):
     @unittest.skipUnless(unix_shell and os.path.exists(unix_shell),
                          'requires a shell')
     @unittest.skipUnless(hasattr(os, 'popen'), "needs os.popen()")
+    @support.requires_subprocess()
     def test_os_popen_iter(self):
         with os.popen("%s -c 'echo \"line1\nline2\nline3\"'"
                       % unix_shell) as popen:
@@ -1173,6 +1175,8 @@ def test_iter_error_when_changing_os_environ_values(self):
     def _test_underlying_process_env(self, var, expected):
         if not (unix_shell and os.path.exists(unix_shell)):
             return
+        elif not support.has_subprocess_support:
+            return
 
         with os.popen(f"{unix_shell} -c 'echo ${var}'") as popen:
             value = popen.read().strip()

Lib/test/test_posix.py

@@ -184,7 +184,7 @@ def test_truncate(self):
         posix.truncate(os_helper.TESTFN, 0)
 
     @unittest.skipUnless(getattr(os, 'execve', None) in os.supports_fd, "test needs execve() to support the fd parameter")
-    @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()")
+    @support.requires_fork()
     def test_fexecve(self):
         fp = os.open(sys.executable, os.O_RDONLY)
         try:
@@ -199,7 +199,7 @@ def test_fexecve(self):
 
 
     @unittest.skipUnless(hasattr(posix, 'waitid'), "test needs posix.waitid()")
-    @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()")
+    @support.requires_fork()
     def test_waitid(self):
         pid = os.fork()
         if pid == 0:
@@ -209,7 +209,7 @@ def test_waitid(self):
             res = posix.waitid(posix.P_PID, pid, posix.WEXITED)
             self.assertEqual(pid, res.si_pid)
 
-    @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()")
+    @support.requires_fork()
     def test_register_at_fork(self):
         with self.assertRaises(TypeError, msg="Positional args not allowed"):
             os.register_at_fork(lambda: None)
@@ -1056,6 +1056,7 @@ def test_getgrouplist(self):
 
     @unittest.skipUnless(hasattr(os, 'getegid'), "test needs os.getegid()")
     @unittest.skipUnless(hasattr(os, 'popen'), "test needs os.popen()")
+    @support.requires_subprocess()
     def test_getgroups(self):
         with os.popen('id -G 2>/dev/null') as idg:
             groups = idg.read().strip()
@@ -1481,7 +1482,7 @@ def test_unlink_dir_fd(self):
                 self.addCleanup(posix.unlink, fullname)
                 raise
 
-    @unittest.skipUnless(os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()")
+    @unittest.skipUnless(hasattr(os, 'mkfifo') and os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()")
     def test_mkfifo_dir_fd(self):
         with self.prepare() as (dir_fd, name, fullname):
             try:

Lib/test/test_pwd.py

@@ -69,7 +69,7 @@ def test_errors(self):
 
         allnames = list(bynames.keys())
         namei = 0
-        fakename = allnames[namei]
+        fakename = allnames[namei] if allnames else "invaliduser"
         while fakename in bynames:
             chars = list(fakename)
             for i in range(len(chars)):

Lib/test/test_pyexpat.py

@@ -12,7 +12,7 @@
 from xml.parsers import expat
 from xml.parsers.expat import errors
 
-from test.support import sortdict
+from test.support import sortdict, is_emscripten
 
 
 class SetAttributeTest(unittest.TestCase):
@@ -466,7 +466,10 @@ def test_exception(self):
                                        "pyexpat.c", "StartElement")
             self.check_traceback_entry(entries[2],
                                        "test_pyexpat.py", "StartElementHandler")
-            if sysconfig.is_python_build() and not (sys.platform == 'win32' and platform.machine() == 'ARM'):
+            if (sysconfig.is_python_build()
+                and not (sys.platform == 'win32' and platform.machine() == 'ARM')
+                and not is_emscripten
+            ):
                 self.assertIn('call_with_frame("StartElement"', entries[1][3])
 
 

Lib/test/test_resource.py

@@ -98,6 +98,7 @@ def test_fsize_toobig(self):
             except (OverflowError, ValueError):
                 pass
 
+    @unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage")
     def test_getrusage(self):
         self.assertRaises(TypeError, resource.getrusage)
         self.assertRaises(TypeError, resource.getrusage, 42, 42)

Lib/test/test_zipfile.py

@@ -22,7 +22,9 @@
 from test.support import script_helper
 from test.support import (findfile, requires_zlib, requires_bz2,
                           requires_lzma, captured_stdout, requires_subprocess)
-from test.support.os_helper import TESTFN, unlink, rmtree, temp_dir, temp_cwd
+from test.support.os_helper import (
+    TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count
+)
 
 
 TESTFN2 = TESTFN + "2"
@@ -2539,14 +2541,14 @@ def test_write_after_read(self):
     def test_many_opens(self):
         # Verify that read() and open() promptly close the file descriptor,
         # and don't rely on the garbage collector to free resources.
+        startcount = fd_count()
         self.make_test_archive(TESTFN2)
         with zipfile.ZipFile(TESTFN2, mode="r") as zipf:
             for x in range(100):
                 zipf.read('ones')
                 with zipf.open('ones') as zopen1:
                     pass
-        with open(os.devnull, "rb") as f:
-            self.assertLess(f.fileno(), 100)
+        self.assertEqual(startcount, fd_count())
 
     def test_write_while_reading(self):
         with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_DEFLATED) as zipf:

Misc/NEWS.d/next/Build/2022-01-31-15-15-08.bpo-40280.r1AYNW.rst

@@ -0,0 +1,9 @@
+Fix wasm32-emscripten test failures and platform issues.
+- Disable syscalls that are not supported or don't work, e.g.
+  wait, getrusage, prlimit, mkfifo, mknod, setres[gu]id, setgroups.
+- Use fd_count to cound open fds.
+- Add more checks for subprocess and fork.
+- Add workarounds for missing _multiprocessing and failing socket.accept().
+- Enable bzip2.
+- Disable large file support.
+- Disable signal.alarm.