bpo-40280: WASM docs and smaller browser builds (GH-32412)

Co-authored-by: Brett Cannon <brett@python.org>

Author: tiran

Doc/library/sys.rst

@@ -1183,7 +1183,9 @@ always available.
    System           ``platform`` value
    ================ ===========================
    AIX              ``'aix'``
+   Emscripten       ``'emscripten'``
    Linux            ``'linux'``
+   WASI             ``'wasi'``
    Windows          ``'win32'``
    Windows/Cygwin   ``'cygwin'``
    macOS            ``'darwin'``

Makefile.pre.in

@@ -812,8 +812,9 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
 # --preload-file turns a relative asset path into an absolute path.
 
 $(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
-                pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py \
-				python.html python.worker.js
+	    $(srcdir)/Tools/wasm/wasm_assets.py \
+	    Makefile pybuilddir.txt Modules/Setup.local \
+	    python.html python.worker.js
 	$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
 	    --builddir . --prefix $(prefix)
 

Tools/wasm/README.md

@@ -9,9 +9,21 @@ possible to build for *wasm32-wasi* out-of-the-box yet.
 
 ## wasm32-emscripten build
 
-Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/)
-tool chain and a build Python interpreter.
-All commands below are relative to a repository checkout.
+Cross compiling to the wasm32-emscripten platform needs the
+[Emscripten](https://emscripten.org/) SDK and a build Python interpreter.
+Emscripten 3.1.8 or newer are recommended. All commands below are relative
+to a repository checkout.
+
+Christian Heimes maintains a container image with Emscripten SDK, Python
+build dependencies, WASI-SDK, wasmtime, and several additional tools.
+
+```
+# Fedora, RHEL, CentOS
+podman run --rm -ti -v $(pwd):/python-wasm/cpython:Z quay.io/tiran/cpythonbuild:emsdk3
+
+# other
+docker run --rm -ti -v $(pwd):/python-wasm/cpython quay.io/tiran/cpythonbuild:emsdk3
+```
 
 ### Compile a build Python interpreter
 
@@ -167,3 +179,77 @@ linker options.
 - pthread support requires WASM threads and SharedArrayBuffer (bulk memory).
   The runtime keeps a pool of web workers around. Each web worker uses
   several file descriptors (eventfd, epoll, pipe).
+
+# Hosting Python WASM builds
+
+The simple REPL terminal uses SharedArrayBuffer. For security reasons
+browsers only provide the feature in secure environents with cross-origin
+isolation. The webserver must send cross-origin headers and correct MIME types
+for the JavaScript and WASM files. Otherwise the terminal will fail to load
+with an error message like ``Browsers disable shared array buffer``.
+
+## Apache HTTP .htaccess
+
+Place a ``.htaccess`` file in the same directory as ``python.wasm``.
+
+```
+# .htaccess
+Header set Cross-Origin-Opener-Policy same-origin
+Header set Cross-Origin-Embedder-Policy require-corp
+
+AddType application/javascript js
+AddType application/wasm wasm
+
+<IfModule mod_deflate.c>
+    AddOutputFilterByType DEFLATE text/html application/javascript application/wasm
+</IfModule>
+```
+
+# Detect WebAssembly builds
+
+## Python code
+
+```# python
+import os, sys
+
+if sys.platform == "emscripten":
+    # Python on Emscripten
+if sys.platform == "wasi":
+    # Python on WASI
+
+if os.name == "posix":
+    # WASM platforms identify as POSIX-like.
+    # Windows does not provide os.uname().
+    machine = os.uname().machine
+    if machine.startswith("wasm"):
+        # WebAssembly (wasm32 or wasm64)
+```
+
+## C code
+
+Emscripten SDK and WASI SDK define several built-in macros. You can dump a
+full list of built-ins with ``emcc -dM -E - < /dev/null`` and
+``/path/to/wasi-sdk/bin/clang -dM -E - < /dev/null``.
+
+```# C
+#ifdef __EMSCRIPTEN__
+    // Python on Emscripten
+#endif
+```
+
+* WebAssembly ``__wasm__`` (also ``__wasm``)
+* wasm32 ``__wasm32__`` (also ``__wasm32``)
+* wasm64 ``__wasm64__``
+* Emscripten ``__EMSCRIPTEN__`` (also ``EMSCRIPTEN``)
+* Emscripten version ``__EMSCRIPTEN_major__``, ``__EMSCRIPTEN_minor__``, ``__EMSCRIPTEN_tiny__``
+* WASI ``__wasi__``
+
+Feature detection flags:
+
+* ``__EMSCRIPTEN_PTHREADS__``
+* ``__EMSCRIPTEN_SHARED_MEMORY__``
+* ``__wasm_simd128__``
+* ``__wasm_sign_ext__``
+* ``__wasm_bulk_memory__``
+* ``__wasm_atomics__``
+* ``__wasm_mutable_globals__``

Tools/wasm/Setup.local.example

@@ -0,0 +1,15 @@
+# Module/Setup.local with reduced stdlib
+*disabled*
+_asyncio
+audioop
+_bz2
+_crypt
+_decimal
+_pickle
+pyexpat _elementtree
+_sha3 _blake2
+_zoneinfo
+xxsubtype
+
+# cjk codecs
+#_multibytecodec _codecs_cn _codecs_hk _codecs_iso2022 _codecs_jp _codecs_kr _codecs_tw

Tools/wasm/wasm_assets.py

@@ -20,7 +20,11 @@
 SRCDIR_LIB = SRCDIR / "Lib"
 
 # sysconfig data relative to build dir.
-SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
+SYSCONFIGDATA = pathlib.PurePath(
+    "build",
+    f"lib.emscripten-wasm32-{sys.version_info.major}.{sys.version_info.minor}",
+    "_sysconfigdata__emscripten_wasm32-emscripten.py",
+)
 
 # Library directory relative to $(prefix).
 WASM_LIB = pathlib.PurePath("lib")
@@ -38,33 +42,44 @@
 OMIT_FILES = (
     # regression tests
     "test/",
-    # user interfaces: TK, curses
-    "curses/",
-    "idlelib/",
-    "tkinter/",
-    "turtle.py",
-    "turtledemo/",
     # package management
     "ensurepip/",
     "venv/",
     # build system
     "distutils/",
     "lib2to3/",
-    # concurrency
-    "concurrent/",
-    "multiprocessing/",
     # deprecated
     "asyncore.py",
     "asynchat.py",
-    # Synchronous network I/O and protocols are not supported; for example,
-    # socket.create_connection() raises an exception:
-    # "BlockingIOError: [Errno 26] Operation in progress".
+    "uu.py",
+    "xdrlib.py",
+    # other platforms
+    "_aix_support.py",
+    "_bootsubprocess.py",
+    "_osx_support.py",
+    # webbrowser
+    "antigravity.py",
+    "webbrowser.py",
+    # Pure Python implementations of C extensions
+    "_pydecimal.py",
+    "_pyio.py",
+    # Misc unused or large files
+    "pydoc_data/",
+    "msilib/",
+)
+
+# Synchronous network I/O and protocols are not supported; for example,
+# socket.create_connection() raises an exception:
+# "BlockingIOError: [Errno 26] Operation in progress".
+OMIT_NETWORKING_FILES = (
     "cgi.py",
     "cgitb.py",
     "email/",
     "ftplib.py",
     "http/",
     "imaplib.py",
+    "mailbox.py",
+    "mailcap.py",
     "nntplib.py",
     "poplib.py",
     "smtpd.py",
@@ -77,26 +92,28 @@
     "urllib/response.py",
     "urllib/robotparser.py",
     "wsgiref/",
-    "xmlrpc/",
-    # dbm / gdbm
-    "dbm/",
-    # other platforms
-    "_aix_support.py",
-    "_bootsubprocess.py",
-    "_osx_support.py",
-    # webbrowser
-    "antigravity.py",
-    "webbrowser.py",
-    # ctypes
-    "ctypes/",
-    # Pure Python implementations of C extensions
-    "_pydecimal.py",
-    "_pyio.py",
-    # Misc unused or large files
-    "pydoc_data/",
-    "msilib/",
 )
 
+OMIT_MODULE_FILES = {
+    "_asyncio": ["asyncio/"],
+    "audioop": ["aifc.py", "sunau.py", "wave.py"],
+    "_crypt": ["crypt.py"],
+    "_curses": ["curses/"],
+    "_ctypes": ["ctypes/"],
+    "_decimal": ["decimal.py"],
+    "_dbm": ["dbm/ndbm.py"],
+    "_gdbm": ["dbm/gnu.py"],
+    "_json": ["json/"],
+    "_multiprocessing": ["concurrent/", "multiprocessing/"],
+    "pyexpat": ["xml/", "xmlrpc/"],
+    "readline": ["rlcompleter.py"],
+    "_sqlite3": ["sqlite3/"],
+    "_ssl": ["ssl.py"],
+    "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"],
+
+    "_zoneinfo": ["zoneinfo/"],
+}
+
 # regression test sub directories
 OMIT_SUBDIRS = (
     "ctypes/test/",
@@ -105,34 +122,59 @@
 )
 
 
-OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
-OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
-
-
-def filterfunc(name: str) -> bool:
-    return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
-
-
 def create_stdlib_zip(
-    args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
+    args: argparse.Namespace,
+    *,
+    optimize: int = 0,
 ) -> None:
-    sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
-    if not sysconfig_data:
-        raise ValueError("No sysconfigdata file found")
+    def filterfunc(name: str) -> bool:
+        return not name.startswith(args.omit_subdirs_absolute)
 
     with zipfile.PyZipFile(
-        args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
+        args.wasm_stdlib_zip, mode="w", compression=args.compression, optimize=optimize
     ) as pzf:
+        if args.compresslevel is not None:
+            pzf.compresslevel = args.compresslevel
+        pzf.writepy(args.sysconfig_data)
         for entry in sorted(args.srcdir_lib.iterdir()):
             if entry.name == "__pycache__":
                 continue
-            if entry in OMIT_ABSOLUTE:
+            if entry in args.omit_files_absolute:
                 continue
             if entry.name.endswith(".py") or entry.is_dir():
                 # writepy() writes .pyc files (bytecode).
                 pzf.writepy(entry, filterfunc=filterfunc)
-        for entry in sysconfig_data:
-            pzf.writepy(entry)
+
+
+def detect_extension_modules(args: argparse.Namespace):
+    modules = {}
+
+    # disabled by Modules/Setup.local ?
+    with open(args.builddir / "Makefile") as f:
+        for line in f:
+            if line.startswith("MODDISABLED_NAMES="):
+                disabled = line.split("=", 1)[1].strip().split()
+                for modname in disabled:
+                    modules[modname] = False
+                break
+
+    # disabled by configure?
+    with open(args.sysconfig_data) as f:
+        data = f.read()
+    loc = {}
+    exec(data, globals(), loc)
+
+    for name, value in loc["build_time_vars"].items():
+        if value not in {"yes", "missing", "disabled", "n/a"}:
+            continue
+        if not name.startswith("MODULE_"):
+            continue
+        if name.endswith(("_CFLAGS", "_DEPS", "_LDFLAGS")):
+            continue
+        modname = name.removeprefix("MODULE_").lower()
+        if modname not in modules:
+            modules[modname] = value == "yes"
+    return modules
 
 
 def path(val: str) -> pathlib.Path:
@@ -147,7 +189,10 @@ def path(val: str) -> pathlib.Path:
     type=path,
 )
 parser.add_argument(
-    "--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
+    "--prefix",
+    help="install prefix",
+    default=pathlib.Path("/usr/local"),
+    type=path,
 )
 
 
@@ -162,6 +207,27 @@ def main():
     args.wasm_stdlib = args.wasm_root / WASM_STDLIB
     args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
 
+    # bpo-17004: zipimport supports only zlib compression.
+    # Emscripten ZIP_STORED + -sLZ4=1 linker flags results in larger file.
+    args.compression = zipfile.ZIP_DEFLATED
+    args.compresslevel = 9
+
+    args.sysconfig_data = args.builddir / SYSCONFIGDATA
+    if not args.sysconfig_data.is_file():
+        raise ValueError(f"sysconfigdata file {SYSCONFIGDATA} missing.")
+
+    extmods = detect_extension_modules(args)
+    omit_files = list(OMIT_FILES)
+    omit_files.extend(OMIT_NETWORKING_FILES)
+    for modname, modfiles in OMIT_MODULE_FILES.items():
+        if not extmods.get(modname):
+            omit_files.extend(modfiles)
+
+    args.omit_files_absolute = {args.srcdir_lib / name for name in omit_files}
+    args.omit_subdirs_absolute = tuple(
+        str(args.srcdir_lib / name) for name in OMIT_SUBDIRS
+    )
+
     # Empty, unused directory for dynamic libs, but required for site initialization.
     args.wasm_dynload.mkdir(parents=True, exist_ok=True)
     marker = args.wasm_dynload / ".empty"
@@ -170,7 +236,7 @@ def main():
     shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
     # The rest of stdlib that's useful in a WASM context.
     create_stdlib_zip(args)
-    size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
+    size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2)
     parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
 
 

Tools/wasm/wasm_webserver.py

@@ -14,6 +14,13 @@
 
 
 class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler):
+    extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy()
+    extensions_map.update(
+        {
+            ".wasm": "application/wasm",
+        }
+    )
+
     def end_headers(self):
         self.send_my_headers()
         super().end_headers()