Inline pthread worker.js file into the main output file

This method has many advantages over the previous method of generating
a separate file:

- Avoids separate output file simplifying deployment.
- Avoids confusing laying of global scopes
- Avoids exporting symbols on the Module simply for visibility within
  the worker file.
- Avoids code duplication
- Avoids the needs to importScripts call, and the node polyfill for
  this.
- Allows optimizers such as closure and JSDCE to operate on the combined
  code.
- `-sSINGLE_FILE` now works with pthreads
- Fewer network requests
- No need for locateFile logic to run on the worker to find the worker.js

The test_pthread_custom_pthread_main_url test was completely removed
since it was testing how worker.js was located and worker.js no longer
exists.

Fixes: emscripten-core#9796

.eslintrc.yml

@@ -22,7 +22,6 @@ ignorePatterns:
   - "src/closure-externs/"
   - "src/embind/"
   - "src/emrun_postjs.js"
-  - "src/worker.js"
   - "src/wasm_worker.js"
   - "src/audio_worklet.js"
   - "src/wasm2js.js"

ChangeLog.md

@@ -23,6 +23,10 @@ See docs/process.md for more on how version tagging works.
 - Enable use of `::` to escape port option separator (#21710)
 - In multi-threaded builds `--extern-pre-js` and `--extern-post-js` code is
   now only run on the main thread, and not on each of the workers. (#21750)
+- Multi-threaded builds no longer generate a separate `.worker.js` file, saving
+  code size and network requests.  This might require updates to build or
+  deployment scripts, since this file is no longer present in the output.
+  (#21701)
 
 3.1.57 - 04/10/24
 -----------------

site/source/docs/porting/pthreads.rst

@@ -146,8 +146,6 @@ The Emscripten implementation for the pthreads API should follow the POSIX stand
 
 - Pthreads + memory growth (``ALLOW_MEMORY_GROWTH``) is especially tricky, see `Wasm design issue #1271 <https://github.com/WebAssembly/design/issues/1271>`_. This currently causes JS accessing the Wasm memory to be slow - but this will likely only be noticeable if the JS does large amounts of memory reads and writes (Wasm runs at full speed, so moving work over can fix this). This also requires that your JS be aware that the HEAP* views may need to be updated - JS code embedded with ``--js-library`` etc will automatically be transformed to use the ``GROWABLE_HEAP_*`` helper functions where ``HEAP*`` are used, but external code that uses ``Module.HEAP*`` directly may encounter problems with views being smaller than memory.
 
-Also note that when compiling code that uses pthreads, an additional JavaScript file ``NAME.worker.js`` is generated alongside the output .js file (where ``NAME`` is the basename of the main file being emitted). That file must be deployed with the rest of the generated code files. By default, ``NAME.worker.js`` will be loaded relative to the main HTML page URL. If it is desirable to load the file from a different location e.g. in a CDN environment, then one can define the ``Module.locateFile(filename)`` function in the main HTML ``Module`` object to return the URL of the target location of the ``NAME.worker.js`` entry point. If this function is not defined in ``Module``, then the default location relative to the main HTML file is used.
-
 .. _Allocator_performance:
 
 Allocator performance

src/library.js

@@ -83,6 +83,9 @@ addToLibrary({
 #if ASSERTIONS || EXIT_RUNTIME
     '$keepRuntimeAlive',
 #endif
+#if PTHREADS
+    '$exitOnMainThread',
+#endif
 #if PTHREADS_DEBUG
     '$runtimeKeepaliveCounter',
 #endif

src/library_async.js

@@ -38,6 +38,8 @@ addToLibrary({
     rewindArguments: {},
 #endif
     instrumentWasmImports(imports) {
+      if (imports.instrumented) return;
+      imports.instrumented = true;
 #if ASYNCIFY_DEBUG
       dbg('asyncify instrumenting imports');
 #endif

src/library_pthread.js

@@ -75,7 +75,7 @@ var LibraryPThread = {
         ) {
           t = _pthread_self();
         }
-        return 'w:' + (Module['workerID'] || 0) + ',t:' + ptrToString(t) + ': ';
+        return 'w:' + (workerID || 0) + ',t:' + ptrToString(t) + ': ';
       }
 
       // Prefix all err()/dbg() messages with the calling thread ID.
@@ -127,16 +127,6 @@ var LibraryPThread = {
     },
 
     initWorker() {
-#if MAYBE_CLOSURE_COMPILER
-      // worker.js is not compiled together with us, and must access certain
-      // things.
-      PThread['receiveObjectTransfer'] = PThread.receiveObjectTransfer;
-      PThread['threadInitTLS'] = PThread.threadInitTLS;
-#if !MINIMAL_RUNTIME
-      PThread['setExitStatus'] = PThread.setExitStatus;
-#endif
-#endif
-
 #if isSymbolNeeded('$noExitRuntime')
       // The default behaviour for pthreads is always to exit once they return
       // from their entry point (or call pthread_exit).  If we set noExitRuntime
@@ -373,16 +363,6 @@ var LibraryPThread = {
       worker.postMessage({
         'cmd': 'load',
         'handlers': handlers,
-        // If the application main .js file was loaded from a Blob, then it is not possible
-        // to access the URL of the current script that could be passed to a Web Worker so that
-        // it could load up the same file. In that case, developer must either deliver the Blob
-        // object in Module['mainScriptUrlOrBlob'], or a URL to it, so that pthread Workers can
-        // independently load up the same main application file.
-        'urlOrBlob': Module['mainScriptUrlOrBlob']
-#if !EXPORT_ES6
-        || _scriptName
-#endif
-        ,
 #if WASM2JS
         // the polyfill WebAssembly.Memory instance has function properties,
         // which will fail in postMessage, so just send a custom object with the
@@ -440,34 +420,47 @@ var LibraryPThread = {
     // Creates a new web Worker and places it in the unused worker pool to wait for its use.
     allocateUnusedWorker() {
       var worker;
-#if MINIMAL_RUNTIME
-      var pthreadMainJs = Module['worker'] || './{{{ PTHREAD_WORKER_FILE }}}';
-#else
+      var workerOptions = {
+#if EXPORT_ES6
+        'type': 'module',
+#endif
+#if ENVIRONMENT_MAY_BE_NODE
+        // This is the way that we signal to the node worker that it is hosting
+        // a pthread.
+        'workerData': 'em-pthread',
+#endif
+#if ENVIRONMENT_MAY_BE_WEB
+        // This is the way that we signal to the Web Worker that it is hosting
+        // a pthread.
+        'name': 'em-pthread',
+#endif
+      };
 #if EXPORT_ES6 && USE_ES6_IMPORT_META
-      // If we're using module output and there's no explicit override, use bundler-friendly pattern.
-      if (!Module['locateFile']) {
+      // If we're using module output, use bundler-friendly pattern.
 #if PTHREADS_DEBUG
-        dbg('Allocating a new web worker from ' + new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url));
+      dbg('Allocating a new web worker from ' + import.meta.url);
 #endif
 #if TRUSTED_TYPES
-        // Use Trusted Types compatible wrappers.
-        if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
-          var p = trustedTypes.createPolicy(
-            'emscripten#workerPolicy1',
-            {
-              createScriptURL: (ignored) => new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url);
-            }
-          );
-          worker = new Worker(p.createScriptURL('ignored'), {type: 'module'});
-        } else
-#endif
-        worker = new Worker(new URL('{{{ PTHREAD_WORKER_FILE }}}', import.meta.url), {type: 'module'});
-      } else {
+      // Use Trusted Types compatible wrappers.
+      if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
+        var p = trustedTypes.createPolicy(
+          'emscripten#workerPolicy1',
+          {
+            createScriptURL: (ignored) => new URL(import.meta.url);
+          }
+        );
+        worker = new Worker(p.createScriptURL('ignored'), workerOptions);
+      } else
 #endif
-      // Allow HTML module to configure the location where the 'worker.js' file will be loaded from,
-      // via Module.locateFile() function. If not specified, then the default URL 'worker.js' relative
-      // to the main html file is loaded.
-      var pthreadMainJs = locateFile('{{{ PTHREAD_WORKER_FILE }}}');
+      worker = new Worker(new URL(import.meta.url), workerOptions);
+#else
+      var pthreadMainJs = _scriptName;
+#if expectToReceiveOnModule('mainScriptUrlOrBlob')
+      // We can't use makeModuleReceiveWithVar here since we want to also
+      // call URL.createObjectURL on the mainScriptUrlOrBlob.
+      if (Module['mainScriptUrlOrBlob']) {
+        pthreadMainJs = URL.createObjectURL(Module['mainScriptUrlOrBlob']);
+      }
 #endif
 #if PTHREADS_DEBUG
       dbg(`Allocating a new web worker from ${pthreadMainJs}`);
@@ -476,14 +469,12 @@ var LibraryPThread = {
       // Use Trusted Types compatible wrappers.
       if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
         var p = trustedTypes.createPolicy('emscripten#workerPolicy2', { createScriptURL: (ignored) => pthreadMainJs });
-        worker = new Worker(p.createScriptURL('ignored'){{{ EXPORT_ES6 ? ", {type: 'module'}" : '' }}});
+        worker = new Worker(p.createScriptURL('ignored'), workerOptions);
       } else
 #endif
-      worker = new Worker(pthreadMainJs{{{ EXPORT_ES6 ? ", {type: 'module'}" : '' }}});
-#if EXPORT_ES6 && USE_ES6_IMPORT_META
-    }
-#endif
-    PThread.unusedWorkers.push(worker);
+      worker = new Worker(pthreadMainJs, workerOptions);
+#endif // EXPORT_ES6 && USE_ES6_IMPORT_META
+      PThread.unusedWorkers.push(worker);
     },
 
     getNewWorker() {

src/library_sdl.js

@@ -2905,14 +2905,14 @@ var LibrarySDL = {
     return SDL.setGetVolume(SDL.music, volume);
   },
 
-  Mix_LoadMUS_RW__docs: '/** @param {number} a1 */',
-  Mix_LoadMUS_RW: 'Mix_LoadWAV_RW',
+  Mix_LoadMUS_RW__deps: ['Mix_LoadWAV_RW'],
+  Mix_LoadMUS_RW: (filename) => _Mix_LoadWAV_RW(filename, 0),
 
   Mix_LoadMUS__deps: ['Mix_LoadMUS_RW', 'SDL_RWFromFile', 'SDL_FreeRW'],
   Mix_LoadMUS__proxy: 'sync',
   Mix_LoadMUS: (filename) => {
     var rwops = _SDL_RWFromFile(filename, 0);
-    var result = _Mix_LoadMUS_RW(rwops, 0);
+    var result = _Mix_LoadMUS_RW(rwops);
     _SDL_FreeRW(rwops);
     return result;
   },

src/parseTools.mjs

@@ -888,21 +888,6 @@ function buildStringArray(array) {
   }
 }
 
-// Generates access to a JS imports scope variable in pthreads worker.js. In MODULARIZE mode these flow into the imports object for the Module.
-// In non-MODULARIZE mode, we can directly access the variables in global scope.
-function makeAsmImportsAccessInPthread(variable) {
-  if (!MINIMAL_RUNTIME) {
-    // Regular runtime uses the name "Module" for both imports and exports.
-    return `Module['${variable}']`;
-  }
-  if (MODULARIZE) {
-    // MINIMAL_RUNTIME uses 'imports' as the name for the imports object in MODULARIZE builds.
-    return `imports['${variable}']`;
-  }
-  // In non-MODULARIZE builds, can access the imports from global scope.
-  return `self.${variable}`;
-}
-
 function _asmjsDemangle(symbol) {
   if (symbol.startsWith('dynCall_')) {
     return symbol;
@@ -1130,7 +1115,6 @@ addToCompileTimeContext({
   hasExportedSymbol,
   implicitSelf,
   isSymbolNeeded,
-  makeAsmImportsAccessInPthread,
   makeDynCall,
   makeEval,
   makeGetValue,

src/postamble.js

@@ -144,8 +144,8 @@ function stackCheckInit() {
 
 #if MAIN_MODULE && PTHREADS
 // Map of modules to be shared with new threads.  This gets populated by the
-// main thread and shared with all new workers.
-var sharedModules = Module['sharedModules'] || [];
+// main thread and shared with all new workers via the initial `load` message.
+var sharedModules = {};
 #endif
 
 #if MAIN_READS_PARAMS

src/postamble_minimal.js

@@ -89,15 +89,6 @@ function initRuntime(wasmExports) {
 
 // Initialize wasm (asynchronous)
 
-var imports = {
-#if MINIFY_WASM_IMPORTED_MODULES
-  'a': wasmImports,
-#else // MINIFY_WASM_IMPORTED_MODULES
-  'env': wasmImports,
-  '{{{ WASI_MODULE_NAME }}}': wasmImports,
-#endif // MINIFY_WASM_IMPORTED_MODULES
-};
-
 // In non-fastcomp non-asm.js builds, grab wasm exports to outer scope
 // for emscripten_get_exported_function() to be able to access them.
 #if LibraryManager.has('library_exports.js')
@@ -112,6 +103,20 @@ var wasmModule;
 <<< WASM_MODULE_EXPORTS_DECLARES >>>
 #endif
 
+#if PTHREADS
+function loadModule() {
+  assignWasmImports();
+#endif
+
+var imports = {
+#if MINIFY_WASM_IMPORTED_MODULES
+  'a': wasmImports,
+#else // MINIFY_WASM_IMPORTED_MODULES
+  'env': wasmImports,
+  '{{{ WASI_MODULE_NAME }}}': wasmImports,
+#endif // MINIFY_WASM_IMPORTED_MODULES
+};
+
 #if MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION
 // https://caniuse.com/#feat=wasm and https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiateStreaming
 // Firefox 52 added Wasm support, but only Firefox 58 added instantiateStreaming.
@@ -252,3 +257,13 @@ WebAssembly.instantiate(Module['wasm'], imports).then((output) => {
 }
 #endif // ASSERTIONS || WASM == 2
 );
+
+#if PTHREADS
+}
+
+if (!ENVIRONMENT_IS_PTHREAD) {
+  // When running in a pthread we delay module loading untill we have
+  // received the module via postMessage
+  loadModule();
+}
+#endif

src/preamble.js

@@ -14,6 +14,10 @@
 // An online HTML version (which may be of a different version of Emscripten)
 //    is up at http://kripken.github.io/emscripten-site/docs/api_reference/preamble.js.html
 
+#if PTHREADS
+#include "runtime_pthread.js"
+#endif
+
 #if RELOCATABLE
 {{{ makeModuleReceiveWithVar('dynamicLibraries', undefined, '[]', true) }}}
 #endif
@@ -897,11 +901,18 @@ function instantiateAsync(binary, binaryFile, imports, callback) {
 }
 #endif // WASM_ASYNC_COMPILATION
 
-// Create the wasm instance.
-// Receives the wasm imports, returns the exports.
-function createWasm() {
+function getWasmImports() {
+#if PTHREADS
+  assignWasmImports();
+#endif
+#if ASYNCIFY && (ASSERTIONS || ASYNCIFY == 2)
+  // instrumenting imports is used in asyncify in two ways: to add assertions
+  // that check for proper import use, and for ASYNCIFY=2 we use them to set up
+  // the Promise API on the import side.
+  Asyncify.instrumentWasmImports(wasmImports);
+#endif
   // prepare imports
-  var info = {
+  return {
 #if MINIFY_WASM_IMPORTED_MODULES
     'a': wasmImports,
 #else // MINIFY_WASM_IMPORTED_MODULES
@@ -915,7 +926,13 @@ function createWasm() {
     'GOT.mem': new Proxy(wasmImports, GOTHandler),
     'GOT.func': new Proxy(wasmImports, GOTHandler),
 #endif
-  };
+  }
+}
+
+// Create the wasm instance.
+// Receives the wasm imports, returns the exports.
+function createWasm() {
+  var info = getWasmImports();
   // Load the wasm module and create an instance of using native support in the JS engine.
   // handle a generated wasm instance, receiving its exports and
   // performing other necessary setup
@@ -1047,21 +1064,6 @@ function createWasm() {
   // Also pthreads and wasm workers initialize the wasm instance through this
   // path.
   if (Module['instantiateWasm']) {
-
-#if USE_OFFSET_CONVERTER
-#if ASSERTIONS
-{{{ runIfWorkerThread("assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object');") }}}
-#endif
-{{{ runIfWorkerThread("wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']);") }}}
-#endif
-
-#if LOAD_SOURCE_MAP
-#if ASSERTIONS
-{{{ runIfWorkerThread("assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object');") }}}
-#endif
-{{{ runIfWorkerThread("wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']);") }}}
-#endif
-
     try {
       return Module['instantiateWasm'](info, receiveInstance);
     } catch(e) {