Inline pthread worker.js file into the main output file (#21701)

This method has many advantages over the previous method of generating a separate file: - Avoids separate output file simplifying deployment. - Avoids confusing laying of global scopes - Avoids exporting symbols on the Module simply for visibility within the worker file. - Avoids code duplication - Avoids the needs to importScripts call, and the node polyfill for this. - Allows optimizers such as closure and JSDCE to operate on the combined code. - `-sSINGLE_FILE` now works with pthreads - Fewer network requests - No need for locateFile logic to run on the worker to find the worker.js The test_pthread_custom_pthread_main_url test was completely removed since it was testing how worker.js was located and worker.js no longer exists. test_pthread_safe_stack depends on `onAbort` being proxied back to the main thread. In this case the `onAbort` handler is injected conditionally in `--pre-js=browser_report.js`. In the previous code this meant that the proxied version took precedence because the pthread handler override was injected first. test_pthread_asan_use_after_free_2_wasmfs depends on `printErr` not being proxied back to the main thread. It is injected unconditionally during `--pre-js`. In the previous code path this means that non-proxied version took precedence because it overrode the incoming pthread handler. Fixes: #9796
emscripten-core · Apr 16, 2024 · 37e1e68 · 37e1e68
1 parent 9f19ec9
commit 37e1e68
Show file tree

Hide file tree

Showing 39 changed files with 598 additions and 645 deletions.
diff --git a/.eslintrc.yml b/.eslintrc.yml
@@ -22,7 +22,6 @@ ignorePatterns:
   - "src/closure-externs/"
   - "src/embind/"
   - "src/emrun_postjs.js"
-  - "src/worker.js"
   - "src/wasm_worker.js"
   - "src/audio_worklet.js"
   - "src/wasm2js.js"

diff --git a/ChangeLog.md b/ChangeLog.md
@@ -23,6 +23,12 @@ See docs/process.md for more on how version tagging works.
 - Enable use of `::` to escape port option separator (#21710)
 - In multi-threaded builds `--extern-pre-js` and `--extern-post-js` code is
   now only run on the main thread, and not on each of the workers. (#21750)
+- Multi-threaded builds no depend on a separate `.worker.js` file.  This saves
+  on code size and network requests.  In order to make this change go smoothly,
+  without breaking build systems that expect a `worker.js`, emscripten will
+  generate an empty `.worker.js` to give folks time to transition their
+  deployment scripts.  In `-sSTRICT` mode, this empty file will not be
+  generated. (#21701)
 
 3.1.57 - 04/10/24
 -----------------

diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst
@@ -146,8 +146,6 @@ The Emscripten implementation for the pthreads API should follow the POSIX stand
 
 - Pthreads + memory growth (``ALLOW_MEMORY_GROWTH``) is especially tricky, see `Wasm design issue #1271 <https://github.com/WebAssembly/design/issues/1271>`_. This currently causes JS accessing the Wasm memory to be slow - but this will likely only be noticeable if the JS does large amounts of memory reads and writes (Wasm runs at full speed, so moving work over can fix this). This also requires that your JS be aware that the HEAP* views may need to be updated - JS code embedded with ``--js-library`` etc will automatically be transformed to use the ``GROWABLE_HEAP_*`` helper functions where ``HEAP*`` are used, but external code that uses ``Module.HEAP*`` directly may encounter problems with views being smaller than memory.
 
-Also note that when compiling code that uses pthreads, an additional JavaScript file ``NAME.worker.js`` is generated alongside the output .js file (where ``NAME`` is the basename of the main file being emitted). That file must be deployed with the rest of the generated code files. By default, ``NAME.worker.js`` will be loaded relative to the main HTML page URL. If it is desirable to load the file from a different location e.g. in a CDN environment, then one can define the ``Module.locateFile(filename)`` function in the main HTML ``Module`` object to return the URL of the target location of the ``NAME.worker.js`` entry point. If this function is not defined in ``Module``, then the default location relative to the main HTML file is used.
-
 .. _Allocator_performance:
 
 Allocator performance

diff --git a/src/closure-externs/minimal_runtime_worker_externs.js b/src/closure-externs/minimal_runtime_worker_externs.js
diff --git a/src/library.js b/src/library.js
@@ -83,6 +83,9 @@ addToLibrary({
 #if ASSERTIONS || EXIT_RUNTIME
     '$keepRuntimeAlive',
 #endif
+#if PTHREADS
+    '$exitOnMainThread',
+#endif
 #if PTHREADS_DEBUG
     '$runtimeKeepaliveCounter',
 #endif

diff --git a/src/library_pthread.js b/src/library_pthread.js
@@ -75,7 +75,7 @@ var LibraryPThread = {
         ) {
           t = _pthread_self();
         }
-        return 'w:' + (Module['workerID'] || 0) + ',t:' + ptrToString(t) + ': ';
+        return 'w:' + workerID + ',t:' + ptrToString(t) + ': ';
       }
 
       // Prefix all err()/dbg() messages with the calling thread ID.
@@ -127,16 +127,6 @@ var LibraryPThread = {
     },
 
     initWorker() {
-#if MAYBE_CLOSURE_COMPILER
-      // worker.js is not compiled together with us, and must access certain
-      // things.
-      PThread['receiveObjectTransfer'] = PThread.receiveObjectTransfer;
-      PThread['threadInitTLS'] = PThread.threadInitTLS;
-#if !MINIMAL_RUNTIME
-      PThread['setExitStatus'] = PThread.setExitStatus;
-#endif
-#endif
-
 #if isSymbolNeeded('$noExitRuntime')
       // The default behaviour for pthreads is always to exit once they return
       // from their entry point (or call pthread_exit).  If we set noExitRuntime
@@ -373,16 +363,6 @@ var LibraryPThread = {
       worker.postMessage({
         'cmd': 'load',
         'handlers': handlers,
-        // If the application main .js file was loaded from a Blob, then it is not possible
-        // to access the URL of the current script that could be passed to a Web Worker so that
-        // it could load up the same file. In that case, developer must either deliver the Blob
-        // object in Module['mainScriptUrlOrBlob'], or a URL to it, so that pthread Workers can
-        // independently load up the same main application file.
-        'urlOrBlob': Module['mainScriptUrlOrBlob']
-#if !EXPORT_ES6
-        || _scriptName
-#endif
-        ,
 #if WASM2JS
         // the polyfill WebAssembly.Memory instance has function properties,
         // which will fail in postMessage, so just send a custom object with the
@@ -440,34 +420,50 @@ var LibraryPThread = {
     // Creates a new web Worker and places it in the unused worker pool to wait for its use.
     allocateUnusedWorker() {
       var worker;
-#if MINIMAL_RUNTIME
-      var pthreadMainJs = Module['worker'] || './{{{ PTHREAD_WORKER_FILE }}}';
-#else
+      var workerOptions = {
+#if EXPORT_ES6
+        'type': 'module',
+#endif
+#if ENVIRONMENT_MAY_BE_NODE
+        // This is the way that we signal to the node worker that it is hosting
+        // a pthread.
+        'workerData': 'em-pthread',
+#endif
+#if ENVIRONMENT_MAY_BE_WEB
+        // This is the way that we signal to the Web Worker that it is hosting
+        // a pthread.
+        'name': 'em-pthread',
+#endif
+      };
 #if EXPORT_ES6 && USE_ES6_IMPORT_META
-      // If we're using module output and there's no explicit override, use bundler-friendly pattern.
-      if (!Module['locateFile']) {
+      // If we're using module output, use bundler-friendly pattern.
 #if PTHREADS_DEBUG
-        dbg('Allocating a new web worker from ' + new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url));
+      dbg('Allocating a new web worker from ' + import.meta.url);
 #endif
 #if TRUSTED_TYPES
-        // Use Trusted Types compatible wrappers.
-        if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
-          var p = trustedTypes.createPolicy(
-            'emscripten#workerPolicy1',
-            {
-              createScriptURL: (ignored) => new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url);
-            }
-          );
-          worker = new Worker(p.createScriptURL('ignored'), {type: 'module'});
-        } else
-#endif
-        worker = new Worker(new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url), {type: 'module'});
-      } else {
+      // Use Trusted Types compatible wrappers.
+      if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
+        var p = trustedTypes.createPolicy(
+          'emscripten#workerPolicy1',
+          {
+            createScriptURL: (ignored) => new URL(import.meta.url);
+          }
+        );
+        worker = new Worker(p.createScriptURL('ignored'), workerOptions);
+      } else
 #endif
-      // Allow HTML module to configure the location where the 'worker.js' file will be loaded from,
-      // via Module.locateFile() function. If not specified, then the default URL 'worker.js' relative
-      // to the main html file is loaded.
-      var pthreadMainJs = locateFile('{{{ PTHREAD_WORKER_FILE }}}');
+      worker = new Worker(new URL(import.meta.url), workerOptions);
+#else
+      var pthreadMainJs = _scriptName;
+#if expectToReceiveOnModule('mainScriptUrlOrBlob')
+      // We can't use makeModuleReceiveWithVar here since we want to also
+      // call URL.createObjectURL on the mainScriptUrlOrBlob.
+      if (Module['mainScriptUrlOrBlob']) {
+        pthreadMainJs = Module['mainScriptUrlOrBlob'];
+        if (typeof pthreadMainJs != 'string') {
+          pthreadMainJs = URL.createObjectURL(pthreadMainJs);
+        }
+      }
 #endif
 #if PTHREADS_DEBUG
       dbg(`Allocating a new web worker from ${pthreadMainJs}`);
@@ -476,14 +472,12 @@ var LibraryPThread = {
       // Use Trusted Types compatible wrappers.
       if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
         var p = trustedTypes.createPolicy('emscripten#workerPolicy2', { createScriptURL: (ignored) => pthreadMainJs });
-        worker = new Worker(p.createScriptURL('ignored'){{{ EXPORT_ES6 ? ", {type: 'module'}" : '' }}});
+        worker = new Worker(p.createScriptURL('ignored'), workerOptions);
       } else
 #endif
-      worker = new Worker(pthreadMainJs{{{ EXPORT_ES6 ? ", {type: 'module'}" : '' }}});
-#if EXPORT_ES6 && USE_ES6_IMPORT_META
-    }
-#endif
-    PThread.unusedWorkers.push(worker);
+      worker = new Worker(pthreadMainJs, workerOptions);
+#endif // EXPORT_ES6 && USE_ES6_IMPORT_META
+      PThread.unusedWorkers.push(worker);
     },
 
     getNewWorker() {

diff --git a/src/library_sdl.js b/src/library_sdl.js
@@ -2907,14 +2907,14 @@ var LibrarySDL = {
     return SDL.setGetVolume(SDL.music, volume);
   },
 
-  Mix_LoadMUS_RW__docs: '/** @param {number} a1 */',
-  Mix_LoadMUS_RW: 'Mix_LoadWAV_RW',
+  Mix_LoadMUS_RW__deps: ['Mix_LoadWAV_RW'],
+  Mix_LoadMUS_RW: (filename) => _Mix_LoadWAV_RW(filename, 0),
 
   Mix_LoadMUS__deps: ['Mix_LoadMUS_RW', 'SDL_RWFromFile', 'SDL_FreeRW'],
   Mix_LoadMUS__proxy: 'sync',
   Mix_LoadMUS: (filename) => {
     var rwops = _SDL_RWFromFile(filename, 0);
-    var result = _Mix_LoadMUS_RW(rwops, 0);
+    var result = _Mix_LoadMUS_RW(rwops);
     _SDL_FreeRW(rwops);
     return result;
   },

diff --git a/src/parseTools.mjs b/src/parseTools.mjs
@@ -888,21 +888,6 @@ function buildStringArray(array) {
   }
 }
 
-// Generates access to a JS imports scope variable in pthreads worker.js. In MODULARIZE mode these flow into the imports object for the Module.
-// In non-MODULARIZE mode, we can directly access the variables in global scope.
-function makeAsmImportsAccessInPthread(variable) {
-  if (!MINIMAL_RUNTIME) {
-    // Regular runtime uses the name "Module" for both imports and exports.
-    return `Module['${variable}']`;
-  }
-  if (MODULARIZE) {
-    // MINIMAL_RUNTIME uses 'imports' as the name for the imports object in MODULARIZE builds.
-    return `imports['${variable}']`;
-  }
-  // In non-MODULARIZE builds, can access the imports from global scope.
-  return `self.${variable}`;
-}
-
 function _asmjsDemangle(symbol) {
   if (symbol.startsWith('dynCall_')) {
     return symbol;
@@ -1130,7 +1115,6 @@ addToCompileTimeContext({
   hasExportedSymbol,
   implicitSelf,
   isSymbolNeeded,
-  makeAsmImportsAccessInPthread,
   makeDynCall,
   makeEval,
   makeGetValue,

diff --git a/src/postamble.js b/src/postamble.js
@@ -144,8 +144,8 @@ function stackCheckInit() {
 
 #if MAIN_MODULE && PTHREADS
 // Map of modules to be shared with new threads.  This gets populated by the
-// main thread and shared with all new workers.
-var sharedModules = Module['sharedModules'] || [];
+// main thread and shared with all new workers via the initial `load` message.
+var sharedModules = {};
 #endif
 
 #if MAIN_READS_PARAMS

diff --git a/src/postamble_minimal.js b/src/postamble_minimal.js
@@ -89,15 +89,6 @@ function initRuntime(wasmExports) {
 
 // Initialize wasm (asynchronous)
 
-var imports = {
-#if MINIFY_WASM_IMPORTED_MODULES
-  'a': wasmImports,
-#else // MINIFY_WASM_IMPORTED_MODULES
-  'env': wasmImports,
-  '{{{ WASI_MODULE_NAME }}}': wasmImports,
-#endif // MINIFY_WASM_IMPORTED_MODULES
-};
-
 // In non-fastcomp non-asm.js builds, grab wasm exports to outer scope
 // for emscripten_get_exported_function() to be able to access them.
 #if LibraryManager.has('library_exports.js')
@@ -112,6 +103,20 @@ var wasmModule;
 <<< WASM_MODULE_EXPORTS_DECLARES >>>
 #endif
 
+#if PTHREADS
+function loadModule() {
+  assignWasmImports();
+#endif
+
+var imports = {
+#if MINIFY_WASM_IMPORTED_MODULES
+  'a': wasmImports,
+#else // MINIFY_WASM_IMPORTED_MODULES
+  'env': wasmImports,
+  '{{{ WASI_MODULE_NAME }}}': wasmImports,
+#endif // MINIFY_WASM_IMPORTED_MODULES
+};
+
 #if MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION
 // https://caniuse.com/#feat=wasm and https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiateStreaming
 // Firefox 52 added Wasm support, but only Firefox 58 added instantiateStreaming.
@@ -252,3 +257,13 @@ WebAssembly.instantiate(Module['wasm'], imports).then((output) => {
 }
 #endif // ASSERTIONS || WASM == 2
 );
+
+#if PTHREADS
+}
+
+if (!ENVIRONMENT_IS_PTHREAD) {
+  // When running in a pthread we delay module loading untill we have
+  // received the module via postMessage
+  loadModule();
+}
+#endif
diff --git a/src/preamble.js b/src/preamble.js
@@ -14,6 +14,10 @@
 // An online HTML version (which may be of a different version of Emscripten)
 //    is up at http://kripken.github.io/emscripten-site/docs/api_reference/preamble.js.html
 
+#if PTHREADS
+#include "runtime_pthread.js"
+#endif
+
 #if RELOCATABLE
 {{{ makeModuleReceiveWithVar('dynamicLibraries', undefined, '[]', true) }}}
 #endif
@@ -897,11 +901,27 @@ function instantiateAsync(binary, binaryFile, imports, callback) {
 }
 #endif // WASM_ASYNC_COMPILATION
 
-// Create the wasm instance.
-// Receives the wasm imports, returns the exports.
-function createWasm() {
+function getWasmImports() {
+#if PTHREADS
+  assignWasmImports();
+#endif
+#if ASYNCIFY && (ASSERTIONS || ASYNCIFY == 2)
+  // instrumenting imports is used in asyncify in two ways: to add assertions
+  // that check for proper import use, and for ASYNCIFY=2 we use them to set up
+  // the Promise API on the import side.
+#if PTHREADS || ASYNCIFY_LAZY_LOAD_CODE
+  // In pthreads builds getWasmImports is called more than once but we only
+  // and the instrument the imports once.
+  if (!wasmImports.__instrumented) {
+    wasmImports.__instrumented = true;
+    Asyncify.instrumentWasmImports(wasmImports);
+  }
+#else
+  Asyncify.instrumentWasmImports(wasmImports);
+#endif
+#endif
   // prepare imports
-  var info = {
+  return {
 #if MINIFY_WASM_IMPORTED_MODULES
     'a': wasmImports,
 #else // MINIFY_WASM_IMPORTED_MODULES
@@ -915,7 +935,13 @@ function createWasm() {
     'GOT.mem': new Proxy(wasmImports, GOTHandler),
     'GOT.func': new Proxy(wasmImports, GOTHandler),
 #endif
-  };
+  }
+}
+
+// Create the wasm instance.
+// Receives the wasm imports, returns the exports.
+function createWasm() {
+  var info = getWasmImports();
   // Load the wasm module and create an instance of using native support in the JS engine.
   // handle a generated wasm instance, receiving its exports and
   // performing other necessary setup
@@ -1047,21 +1073,6 @@ function createWasm() {
   // Also pthreads and wasm workers initialize the wasm instance through this
   // path.
   if (Module['instantiateWasm']) {
-
-#if USE_OFFSET_CONVERTER
-#if ASSERTIONS
-{{{ runIfWorkerThread("assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object');") }}}
-#endif
-{{{ runIfWorkerThread("wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']);") }}}
-#endif
-
-#if LOAD_SOURCE_MAP
-#if ASSERTIONS
-{{{ runIfWorkerThread("assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object');") }}}
-#endif
-{{{ runIfWorkerThread("wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']);") }}}
-#endif
-
     try {
       return Module['instantiateWasm'](info, receiveInstance);
     } catch(e) {