diff --git a/emcc.py b/emcc.py index 1e9f7753ec547..71f5a0a4a1357 100755 --- a/emcc.py +++ b/emcc.py @@ -2076,7 +2076,10 @@ def phase_linker_setup(options, state, newargs): assert not settings.SIDE_MODULE if settings.MAIN_MODULE == 1: settings.INCLUDE_FULL_LIBRARY = 1 + # Called from preamble.js once the main module is instantiated. settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$loadDylibs'] + if settings.STACK_OVERFLOW_CHECK == 2: + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$setDylinkStackLimits'] settings.REQUIRED_EXPORTS += ['malloc'] if settings.MAIN_MODULE == 1 or settings.SIDE_MODULE == 1: diff --git a/src/library_dylink.js b/src/library_dylink.js index 05155ad9f3aeb..04bff62d05ec8 100644 --- a/src/library_dylink.js +++ b/src/library_dylink.js @@ -25,7 +25,7 @@ var LibraryDylink = { // than just running the promises in parallel, this makes a chain of // promises to run in series. wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( - () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then( + () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true}, name)).then( (exports) => { #if DYLINK_DEBUG dbg(`registering preloadedWasm: ${name}`); @@ -359,7 +359,12 @@ var LibraryDylink = { loadedLibsByName: {}, // handle -> dso; Used by dlsym loadedLibsByHandle: {}, - init: () => newDSO('__main__', {{{ cDefs.RTLD_DEFAULT }}}, wasmImports), + init: () => { +#if ASSERTIONS + assert(wasmImports); +#endif + newDSO('__main__', {{{ cDefs.RTLD_DEFAULT }}}, wasmImports); + }, }, $dlSetError__internal: true, @@ -611,6 +616,7 @@ var LibraryDylink = { // promise that resolves to its exports if the loadAsync flag is set. $loadWebAssemblyModule__docs: ` /** + * @param {string=} libName * @param {Object=} localScope * @param {number=} handle */`, @@ -622,7 +628,10 @@ var LibraryDylink = { '$currentModuleWeakSymbols', '$alignMemory', '$zeroMemory', '$updateTableMap', ], - $loadWebAssemblyModule: function(binary, flags, localScope, handle) { + $loadWebAssemblyModule: function(binary, flags, libName, localScope, handle) { +#if DYLINK_DEBUG + dbg(`loadWebAssemblyModule: ${libName}`); +#endif var metadata = getDylinkMetadata(binary); currentModuleWeakSymbols = metadata.weakImports; #if ASSERTIONS @@ -745,10 +754,20 @@ var LibraryDylink = { '{{{ WASI_MODULE_NAME }}}': proxy, }; - function postInstantiation(instance) { + function postInstantiation(module, instance) { #if ASSERTIONS // the table should be unchanged assert(wasmTable === originalTable); +#endif +#if PTHREADS + if (!ENVIRONMENT_IS_PTHREAD && libName) { +#if DYLINK_DEBUG + dbg(`registering sharedModules: ${libName}`) +#endif + // cache all loaded modules in `sharedModules`, which gets passed + // to new workers when they are created. + sharedModules[libName] = module; + } #endif // add new entries to functionsInTableMap updateTableMap(tableBase, metadata.tableSize); @@ -761,12 +780,11 @@ var LibraryDylink = { } #if STACK_OVERFLOW_CHECK >= 2 if (moduleExports['__set_stack_limits']) { -#if PTHREADS // When we are on an uninitialized pthread we delay calling // __set_stack_limits until $setDylinkStackLimits. - if (!ENVIRONMENT_IS_PTHREAD || runtimeInitialized) -#endif - moduleExports['__set_stack_limits']({{{ to64('_emscripten_stack_get_base()') }}}, {{{ to64('_emscripten_stack_get_end()') }}}); + if (runtimeInitialized) { + moduleExports['__set_stack_limits']({{{ to64('_emscripten_stack_get_base()') }}}, {{{ to64('_emscripten_stack_get_end()') }}}); + } } #endif @@ -839,16 +857,16 @@ var LibraryDylink = { if (flags.loadAsync) { if (binary instanceof WebAssembly.Module) { var instance = new WebAssembly.Instance(binary, info); - return Promise.resolve(postInstantiation(instance)); + return Promise.resolve(postInstantiation(binary, instance)); } return WebAssembly.instantiate(binary, info).then( - (result) => postInstantiation(result.instance) + (result) => postInstantiation(result.module, result.instance) ); } var module = binary instanceof WebAssembly.Module ? binary : new WebAssembly.Module(binary); var instance = new WebAssembly.Instance(module, info); - return postInstantiation(instance); + return postInstantiation(module, instance); } // now load needed libraries and the module itself. @@ -864,7 +882,7 @@ var LibraryDylink = { return loadModule(); }, -#if STACK_OVERFLOW_CHECK >= 2 && PTHREADS +#if STACK_OVERFLOW_CHECK >= 2 // With PTHREADS we load libraries before we are running a pthread and // therefore before we have a stack. Instead we delay calling // `__set_stack_limits` until we start running a thread. We also need to call @@ -877,7 +895,7 @@ var LibraryDylink = { #endif var lib = LDSO.loadedLibsByName[name]; if (lib.exports['__set_stack_limits']) { - lib.exports['__set_stack_limits'](stackTop, stackMax); + lib.exports['__set_stack_limits']({{{ to64("stackTop") }}}, {{{ to64("stackMax") }}}); } } }, @@ -965,6 +983,16 @@ var LibraryDylink = { // libName -> libData function loadLibData() { +#if PTHREADS + var sharedMod = sharedModules[libName]; +#if DYLINK_DEBUG + dbg(`checking sharedModules: ${libName}: ${sharedMod ? 'found' : 'not found'}`); +#endif + if (sharedMod) { + return flags.loadAsync ? Promise.resolve(sharedMod) : sharedMod; + } +#endif + // for wasm, we can use fetch for async, but for fs mode we can only imitate it if (handle) { var data = {{{ makeGetValue('handle', C_STRUCTS.dso.file_data, '*') }}}; @@ -1004,10 +1032,10 @@ var LibraryDylink = { // module not preloaded - load lib data and create new module from it if (flags.loadAsync) { - return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, localScope, handle)); + return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, libName, localScope, handle)); } - return loadWebAssemblyModule(loadLibData(), flags, localScope, handle); + return loadWebAssemblyModule(loadLibData(), flags, libName, localScope, handle); } // module for lib is loaded - update the dso & global namespace @@ -1040,9 +1068,6 @@ var LibraryDylink = { $loadDylibs__internal: true, $loadDylibs__deps: ['$loadDynamicLibrary', '$reportUndefinedSymbols'], $loadDylibs: function() { -#if DYLINK_DEBUG - dbg(`loadDylibs: ${dynamicLibraries}`); -#endif if (!dynamicLibraries.length) { #if DYLINK_DEBUG dbg('loadDylibs: no libraries to preload'); @@ -1051,6 +1076,10 @@ var LibraryDylink = { return; } +#if DYLINK_DEBUG + dbg(`loadDylibs: ${dynamicLibraries}`); +#endif + // Load binaries asynchronously addRunDependency('loadDylibs'); dynamicLibraries.reduce((chain, lib) => { diff --git a/src/library_pthread.js b/src/library_pthread.js index 3ead63051ca96..fa7b254d0cea3 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -105,6 +105,14 @@ var LibraryPThread = { PThread.allocateUnusedWorker(); } #endif +#if !MINIMAL_RUNTIME + // MINIMAL_RUNTIME takes care of calling loadWasmModuleToAllWorkers + // in postamble_minimal.js + addOnPreRun(() => { + addRunDependency('loading-workers') + PThread.loadWasmModuleToAllWorkers(() => removeRunDependency('loading-workers')); + }); +#endif #if MAIN_MODULE PThread.outstandingPromises = {}; // Finished threads are threads that have finished running but we not yet @@ -402,7 +410,9 @@ var LibraryPThread = { 'wasmOffsetConverter': wasmOffsetConverter, #endif #if MAIN_MODULE - 'dynamicLibraries': Module['dynamicLibraries'], + // Shared all modules that have been loaded so far. New workers + // won't start running threads until these are all loaded. + 'sharedModules': sharedModules, #endif #if ASSERTIONS 'workerID': worker.workerID, @@ -423,6 +433,7 @@ var LibraryPThread = { ) { return onMaybeReady(); } + let pthreadPoolReady = Promise.all(PThread.unusedWorkers.map(PThread.loadWasmModuleToWorker)); #if PTHREAD_POOL_DELAY_LOAD // PTHREAD_POOL_DELAY_LOAD means we want to proceed synchronously without diff --git a/src/parseTools.js b/src/parseTools.js index 167767e0f2915..12393c071ae21 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -746,7 +746,7 @@ function makeRemovedModuleAPIAssert(moduleName, localName) { function checkReceiving(name) { // ALL_INCOMING_MODULE_JS_API contains all valid incoming module API symbols // so calling makeModuleReceive* with a symbol not in this list is an error - assert(ALL_INCOMING_MODULE_JS_API.has(name)); + assert(ALL_INCOMING_MODULE_JS_API.has(name), `${name} is not part of INCOMING_MODULE_JS_API`); } // Make code to receive a value on the incoming Module object. diff --git a/src/postamble.js b/src/postamble.js index 8e01e640c3516..d235e9da81d4b 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -131,11 +131,10 @@ function stackCheckInit() { } #endif -#if RELOCATABLE -var dylibsLoaded = false; -#if '$LDSO' in addedLibraryItems -LDSO.init(); -#endif +#if MAIN_MODULE && PTHREADS +// Map of modules to be shared with new threads. This gets populated by the +// main thread an shared with all new workers. +var sharedModules = Module['sharedModules'] ||[]; #endif #if MAIN_READS_PARAMS @@ -158,25 +157,8 @@ function run() { stackCheckInit(); #endif -#if RELOCATABLE - if (!dylibsLoaded) { - // Loading of dynamic libraries needs to happen on each thread, so we can't - // use the normal __ATPRERUN__ mechanism. -#if MAIN_MODULE - loadDylibs(); -#else - reportUndefinedSymbols(); -#endif - dylibsLoaded = true; - - // Loading dylibs can add run dependencies. - if (runDependencies > 0) { -#if RUNTIME_DEBUG - dbg('loadDylibs added run() dependencies, not running yet'); -#endif - return; - } - } +#if RELOCATABLE && !MAIN_MODULE + reportUndefinedSymbols(); #endif #if WASM_WORKERS diff --git a/src/preamble.js b/src/preamble.js index 70097ae03387c..0104eb60faa9f 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -234,9 +234,12 @@ function initRuntime() { #if STACK_OVERFLOW_CHECK >= 2 #if RUNTIME_DEBUG - dbg('__set_stack_limits: ' + _emscripten_stack_get_base() + ', ' + _emscripten_stack_get_end()); + dbg(`__set_stack_limits: ${ptrToString(_emscripten_stack_get_base())}, ${ptrToString(_emscripten_stack_get_end())}`); #endif ___set_stack_limits(_emscripten_stack_get_base(), _emscripten_stack_get_end()); +#if MAIN_MODULE + setDylinkStackLimits(_emscripten_stack_get_base(), _emscripten_stack_get_end()); +#endif #endif #if RELOCATABLE callRuntimeCallbacks(__RELOC_FUNCS__); @@ -980,6 +983,10 @@ function createWasm() { } #endif mergeLibSymbols(exports, 'main') +#if '$LDSO' in addedLibraryItems + LDSO.init(); +#endif + loadDylibs(); #endif #if MEMORY64 @@ -1047,13 +1054,7 @@ function createWasm() { // We now have the Wasm module loaded up, keep a reference to the compiled module so we can post it to the workers. wasmModule = module; #endif - -#if PTHREADS - PThread.loadWasmModuleToAllWorkers(() => removeRunDependency('wasm-instantiate')); -#else // singlethreaded build: removeRunDependency('wasm-instantiate'); -#endif // ~PTHREADS - return exports; } // wait for the pthread pool (if any) diff --git a/src/worker.js b/src/worker.js index 4c8fe3cebc4f7..14b78fde7a191 100644 --- a/src/worker.js +++ b/src/worker.js @@ -81,7 +81,7 @@ var out = () => { throw 'out() is not defined in worker.js.'; } #endif var err = threadPrintErr; self.alert = threadAlert; -#if RUNTIME_DEBUG +#if ASSERTIONS || RUNTIME_DEBUG var dbg = threadPrintErr; #endif @@ -151,7 +151,10 @@ function handleMessage(e) { #endif // MINIMAL_RUNTIME #if MAIN_MODULE - Module['dynamicLibraries'] = e.data.dynamicLibraries; + Module['sharedModules'] = e.data.sharedModules; +#if RUNTIME_DEBUG + dbg(`received ${Object.keys(e.data.sharedModules).length} shared modules: ${Object.keys(e.data.sharedModules)}`); +#endif #endif // Use `const` here to ensure that the variable is scoped only to diff --git a/test/other/metadce/test_metadce_hello_dylink.jssize b/test/other/metadce/test_metadce_hello_dylink.jssize index 5da2a4f7c562e..62b62ad8bcd45 100644 --- a/test/other/metadce/test_metadce_hello_dylink.jssize +++ b/test/other/metadce/test_metadce_hello_dylink.jssize @@ -1 +1 @@ -15060 +15032 diff --git a/test/other/metadce/test_metadce_minimal_pthreads.jssize b/test/other/metadce/test_metadce_minimal_pthreads.jssize index 13dac5d04b508..51cd2147d6059 100644 --- a/test/other/metadce/test_metadce_minimal_pthreads.jssize +++ b/test/other/metadce/test_metadce_minimal_pthreads.jssize @@ -1 +1 @@ -15415 +15456 diff --git a/test/other/test_unoptimized_code_size.js.size b/test/other/test_unoptimized_code_size.js.size index cc84a949cd741..068c130d5b5bd 100644 --- a/test/other/test_unoptimized_code_size.js.size +++ b/test/other/test_unoptimized_code_size.js.size @@ -1 +1 @@ -59630 +59629 diff --git a/test/other/test_unoptimized_code_size_no_asserts.js.size b/test/other/test_unoptimized_code_size_no_asserts.js.size index 5ff8d77a15fd1..2a7a3ea68f817 100644 --- a/test/other/test_unoptimized_code_size_no_asserts.js.size +++ b/test/other/test_unoptimized_code_size_no_asserts.js.size @@ -1 +1 @@ -33256 +33255 diff --git a/test/other/test_unoptimized_code_size_strict.js.size b/test/other/test_unoptimized_code_size_strict.js.size index 7b6c1d7f7e426..bf3a0db19d3b5 100644 --- a/test/other/test_unoptimized_code_size_strict.js.size +++ b/test/other/test_unoptimized_code_size_strict.js.size @@ -1 +1 @@ -58572 +58571 diff --git a/test/test_core.py b/test/test_core.py index 7abd9ccf7653b..218d175fa4c34 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -9474,34 +9474,39 @@ def test_pthread_dylink_main_module_1(self): self.do_runf(test_file('hello_world.c')) @needs_dylink - @node_pthreads - def test_Module_dynamicLibraries_pthreads(self): + @parameterized({ + '': ([],), + 'pthreads': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME', '-pthread', '-Wno-experimental'],) + }) + def test_Module_dynamicLibraries(self, args): # test that Module.dynamicLibraries works with pthreads - self.emcc_args += ['-pthread', '-Wno-experimental'] + self.emcc_args += args self.emcc_args += ['--pre-js', 'pre.js'] - self.set_setting('PROXY_TO_PTHREAD') - self.set_setting('EXIT_RUNTIME') # This test is for setting dynamicLibraries at runtime so we don't # want emscripten loading `liblib.so` automatically (which it would # do without this setting. self.set_setting('NO_AUTOLOAD_DYLIBS') create_file('pre.js', ''' - if (typeof importScripts == 'undefined') { // !ENVIRONMENT_IS_WORKER - // Load liblib.so by default on non-workers - Module['dynamicLibraries'] = ['liblib.so']; - } else { - // Verify whether the main thread passes Module.dynamicLibraries to the worker - assert(Module['dynamicLibraries'].includes('liblib.so')); - } + Module['dynamicLibraries'] = ['liblib.so']; ''') + if args: + self.setup_node_pthreads() + create_file('post.js', ''' + if (ENVIRONMENT_IS_PTHREAD) { + err('sharedModules: ' + Object.keys(sharedModules)); + assert('liblib.so' in sharedModules); + } + ''') + self.emcc_args += ['--post-js', 'post.js'] + self.dylink_test( r''' #include int side(); int main() { - printf("result is %d", side()); + printf("result is %d\n", side()); return 0; } ''', diff --git a/test/test_other.py b/test/test_other.py index 3cbe043529f50..640ff3a11fdf5 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -2331,7 +2331,8 @@ def test_undefined_symbols(self, action): print('checking "%s" %s' % (args, value)) extra = ['-s', action + '_ON_UNDEFINED_SYMBOLS=%d' % value] if action else [] proc = self.run_process([EMXX, '-sUSE_SDL', 'main.cpp'] + extra + args, stderr=PIPE, check=False) - print(proc.stderr) + if common.EMTEST_VERBOSE: + print(proc.stderr) if value or action is None: # The default is that we error in undefined symbols self.assertContained('undefined symbol: something', proc.stderr) @@ -13470,12 +13471,20 @@ def test_preload_module(self, args): struct stat statbuf; assert(stat("/library.so", &statbuf) == 0); - // Check that it was preloaded + // Check that it was preloaded. + // The preloading actually only happens on the main thread where the filesystem + // lives. On worker threads the module object is shared via preloadedModules. if (emscripten_is_main_runtime_thread()) { int found = EM_ASM_INT( return preloadedWasm['/library.so'] !== undefined; ); assert(found); + } else { + int found = EM_ASM_INT( + err(sharedModules); + return sharedModules['/library.so'] !== undefined; + ); + assert(found); } void *lib_handle = dlopen("/library.so", RTLD_NOW); assert(lib_handle);