Add support for -sEXPORT_ES6/*.mjs on Node.js

ChangeLog.md

@@ -31,6 +31,7 @@ See docs/process.md for more on how version tagging works.
   overflow will trap rather corrupting global data first).  This should not
   be a user-visible change (unless your program does something very odd such
   depending on the specific location of stack data in memory). (#18154)
+- Add support for `-sEXPORT_ES6`/`*.mjs` on Node.js. (#17915)
 
 3.1.25 - 11/08/22
 -----------------

emcc.py

@@ -2338,11 +2338,17 @@ def check_memory_setting(setting):
   if 'MAXIMUM_MEMORY' in user_settings and not settings.ALLOW_MEMORY_GROWTH:
     diagnostics.warning('unused-command-line-argument', 'MAXIMUM_MEMORY is only meaningful with ALLOW_MEMORY_GROWTH')
 
-  if settings.EXPORT_ES6 and not settings.MODULARIZE:
-    # EXPORT_ES6 requires output to be a module
-    if 'MODULARIZE' in user_settings:
-      exit_with_error('EXPORT_ES6 requires MODULARIZE to be set')
-    settings.MODULARIZE = 1
+  if settings.EXPORT_ES6:
+    if not settings.MODULARIZE:
+      # EXPORT_ES6 requires output to be a module
+      if 'MODULARIZE' in user_settings:
+        exit_with_error('EXPORT_ES6 requires MODULARIZE to be set')
+      settings.MODULARIZE = 1
+    if shared.target_environment_may_be('node') and not settings.USE_ES6_IMPORT_META:
+      # EXPORT_ES6 + ENVIRONMENT=*node* requires the use of import.meta.url
+      if 'USE_ES6_IMPORT_META' in user_settings:
+        exit_with_error('EXPORT_ES6 and ENVIRONMENT=*node* requires USE_ES6_IMPORT_META to be set')
+      settings.USE_ES6_IMPORT_META = 1
 
   if settings.MODULARIZE and not settings.DECLARE_ASM_MODULE_EXPORTS:
     # When MODULARIZE option is used, currently requires declaring all module exports
@@ -3103,13 +3109,17 @@ def phase_final_emitting(options, state, target, wasm_target, memfile):
     # mode)
     final_js = building.closure_compiler(final_js, pretty=False, advanced=False, extra_closure_args=options.closure_args)
 
-  # Unmangle previously mangled `import.meta` references in both main code and libraries.
+  # Unmangle previously mangled `import.meta` and `await import` references in
+  # both main code and libraries.
   # See also: `preprocess` in parseTools.js.
   if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META:
     src = read_file(final_js)
     final_js += '.esmeta.js'
-    write_file(final_js, src.replace('EMSCRIPTEN$IMPORT$META', 'import.meta'))
-    save_intermediate('es6-import-meta')
+    write_file(final_js, src
+               .replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
+               .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import'))
+    shared.get_temp_files().note(final_js)
+    save_intermediate('es6-module')
 
   # Apply pre and postjs files
   if options.extern_pre_js or options.extern_post_js:
@@ -3681,26 +3691,49 @@ def preprocess_wasm2js_script():
     write_file(final_js, js)
 
 
+def node_es6_imports():
+  if not settings.EXPORT_ES6 or not shared.target_environment_may_be('node'):
+    return ''
+
+  # Multi-environment builds uses `await import` in `shell.js`
+  if shared.target_environment_may_be('web'):
+    return ''
+
+  # Use static import declaration if we only target Node.js
+  return '''
+import { createRequire } from 'module';
+const require = createRequire(import.meta.url);
+'''
+
+
 def modularize():
   global final_js
   logger.debug(f'Modularizing, assigning to var {settings.EXPORT_NAME}')
   src = read_file(final_js)
 
+  # Multi-environment ES6 builds require an async function
+  async_emit = ''
+  if settings.EXPORT_ES6 and \
+     shared.target_environment_may_be('node') and \
+     shared.target_environment_may_be('web'):
+    async_emit = 'async '
+
   return_value = settings.EXPORT_NAME
   if settings.WASM_ASYNC_COMPILATION:
     return_value += '.ready'
   if not settings.EXPORT_READY_PROMISE:
     return_value = '{}'
 
   src = '''
-function(%(EXPORT_NAME)s) {
+%(maybe_async)sfunction(%(EXPORT_NAME)s) {
   %(EXPORT_NAME)s = %(EXPORT_NAME)s || {};
 
 %(src)s
 
   return %(return_value)s
 }
 ''' % {
+    'maybe_async': async_emit,
     'EXPORT_NAME': settings.EXPORT_NAME,
     'src': src,
     'return_value': return_value
@@ -3711,24 +3744,25 @@ def modularize():
     # document.currentScript, so a simple export declaration is enough.
     src = 'var %s=%s' % (settings.EXPORT_NAME, src)
   else:
-    script_url_node = ""
+    script_url_node = ''
     # When MODULARIZE this JS may be executed later,
     # after document.currentScript is gone, so we save it.
     # In EXPORT_ES6 + USE_PTHREADS the 'thread' is actually an ES6 module webworker running in strict mode,
     # so doesn't have access to 'document'. In this case use 'import.meta' instead.
     if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META:
-      script_url = "import.meta.url"
+      script_url = 'import.meta.url'
     else:
       script_url = "typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined"
       if shared.target_environment_may_be('node'):
         script_url_node = "if (typeof __filename !== 'undefined') _scriptDir = _scriptDir || __filename;"
-    src = '''
+    src = '''%(node_imports)s
 var %(EXPORT_NAME)s = (() => {
   var _scriptDir = %(script_url)s;
   %(script_url_node)s
   return (%(src)s);
 })();
 ''' % {
+      'node_imports': node_es6_imports(),
       'EXPORT_NAME': settings.EXPORT_NAME,
       'script_url': script_url,
       'script_url_node': script_url_node,

src/closure-externs/closure-externs.js

@@ -11,8 +11,12 @@
  * The closure_compiler() method in tools/shared.py refers to this file when calling closure.
  */
 
-// Special placeholder for `import.meta`.
+// Special placeholder for `import.meta` and `await import`.
 var EMSCRIPTEN$IMPORT$META;
+var EMSCRIPTEN$AWAIT$IMPORT;
+
+// Don't minify createRequire
+var createRequire;
 
 // Closure externs used by library_sockfs.js
 

src/node_shell_read.js

@@ -4,29 +4,16 @@
  * SPDX-License-Identifier: MIT
  */
 
-// These modules will usually be used on Node.js. Load them eagerly to avoid
-// the complexity of lazy-loading. However, for now we must guard on require()
-// actually existing: if the JS is put in a .mjs file (ES6 module) and run on
-// node, then we'll detect node as the environment and get here, but require()
-// does not exist (since ES6 modules should use |import|). If the code actually
-// uses the node filesystem then it will crash, of course, but in the case of
-// code that never uses it we don't want to crash here, so the guarding if lets
-// such code work properly. See discussion in
-// https://github.com/emscripten-core/emscripten/pull/17851
-var fs, nodePath;
-if (typeof require === 'function') {
-  fs = require('fs');
-  nodePath = require('path');
-}
-
 read_ = (filename, binary) => {
 #if SUPPORT_BASE64_EMBEDDING
   var ret = tryParseAsDataURI(filename);
   if (ret) {
     return binary ? ret : ret.toString();
   }
 #endif
-  filename = nodePath['normalize'](filename);
+  // We need to re-wrap `file://` strings to URLs. Normalizing isn't
+  // necessary in that case, the path should already be absolute.
+  filename = isFileURI(filename) ? new URL(filename) : nodePath.normalize(filename);
   return fs.readFileSync(filename, binary ? undefined : 'utf8');
 };
 
@@ -48,7 +35,8 @@ readAsync = (filename, onload, onerror) => {
     onload(ret);
   }
 #endif
-  filename = nodePath['normalize'](filename);
+  // See the comment in the `read_` function.
+  filename = isFileURI(filename) ? new URL(filename) : nodePath.normalize(filename);
   fs.readFile(filename, function(err, data) {
     if (err) onerror(err);
     else onload(data.buffer);

src/parseTools.js

@@ -37,10 +37,12 @@ function processMacros(text) {
 function preprocess(text, filenameHint) {
   if (EXPORT_ES6 && USE_ES6_IMPORT_META) {
     // `eval`, Terser and Closure don't support module syntax; to allow it,
-    // we need to temporarily replace `import.meta` usages with placeholders
-    // during preprocess phase, and back after all the other ops.
+    // we need to temporarily replace `import.meta` and `await import` usages
+    // with placeholders during preprocess phase, and back after all the other ops.
     // See also: `phase_final_emitting` in emcc.py.
-    text = text.replace(/\bimport\.meta\b/g, 'EMSCRIPTEN$IMPORT$META');
+    text = text
+      .replace(/\bimport\.meta\b/g, 'EMSCRIPTEN$IMPORT$META')
+      .replace(/\bawait import\b/g, 'EMSCRIPTEN$AWAIT$IMPORT');
   }
 
   const IGNORE = 0;

src/preamble.js

@@ -641,7 +641,7 @@ if (Module['locateFile']) {
 #if EXPORT_ES6 && USE_ES6_IMPORT_META && !SINGLE_FILE // in single-file mode, repeating WASM_BINARY_FILE would emit the contents again
 } else {
   // Use bundler-friendly `new URL(..., import.meta.url)` pattern; works in browsers too.
-  wasmBinaryFile = new URL('{{{ WASM_BINARY_FILE }}}', import.meta.url).toString();
+  wasmBinaryFile = new URL('{{{ WASM_BINARY_FILE }}}', import.meta.url).href;
 }
 #endif
 

src/settings.js

@@ -1236,7 +1236,8 @@ var EXPORT_ES6 = false;
 
 // Use the ES6 Module relative import feature 'import.meta.url'
 // to auto-detect WASM Module path.
-// It might not be supported on old browsers / toolchains
+// It might not be supported on old browsers / toolchains. This setting
+// may not be disabled when Node.js is targeted (-sENVIRONMENT=*node*).
 // [link]
 var USE_ES6_IMPORT_META = true;
 

src/shell.js

@@ -133,9 +133,6 @@ var ENVIRONMENT_IS_WASM_WORKER = Module['$ww'];
 #if SHARED_MEMORY && !MODULARIZE
 // In MODULARIZE mode _scriptDir needs to be captured already at the very top of the page immediately when the page is parsed, so it is generated there
 // before the page load. In non-MODULARIZE modes generate it here.
-#if EXPORT_ES6
-var _scriptDir = import.meta.url;
-#else
 var _scriptDir = (typeof document != 'undefined' && document.currentScript) ? document.currentScript.src : undefined;
 
 if (ENVIRONMENT_IS_WORKER) {
@@ -146,8 +143,7 @@ else if (ENVIRONMENT_IS_NODE) {
   _scriptDir = __filename;
 }
 #endif // ENVIRONMENT_MAY_BE_NODE
-#endif
-#endif
+#endif // SHARED_MEMORY && !MODULARIZE
 
 // `/` should be present at the end if `scriptDirectory` is not empty
 var scriptDirectory = '';
@@ -193,10 +189,31 @@ if (ENVIRONMENT_IS_NODE) {
   if (typeof process == 'undefined' || !process.release || process.release.name !== 'node') throw new Error('not compiled for this environment (did you build to HTML and try to run it not on the web, or set ENVIRONMENT to something - like node - and run it someplace else - like on the web?)');
 #endif
 #endif
+  // `require()` is no-op in an ESM module, use `createRequire()` to construct
+  // the require()` function.  This is only necessary for multi-environment
+  // builds, `-sENVIRONMENT=node` emits a static import declaration instead.
+  // TODO: Swap all `require()`'s with `import()`'s?
+#if EXPORT_ES6 && ENVIRONMENT_MAY_BE_WEB
+  const { createRequire } = await import('module');
+  /** @suppress{duplicate} */
+  var require = createRequire(import.meta.url);
+#endif
+  // These modules will usually be used on Node.js. Load them eagerly to avoid
+  // the complexity of lazy-loading.
+  var fs = require('fs');
+  var nodePath = require('path');
+
   if (ENVIRONMENT_IS_WORKER) {
-    scriptDirectory = require('path').dirname(scriptDirectory) + '/';
+    scriptDirectory = nodePath.dirname(scriptDirectory) + '/';
   } else {
+#if EXPORT_ES6
+    // EXPORT_ES6 + ENVIRONMENT_IS_NODE always requires use of import.meta.url,
+    // since there's no way getting the current absolute path of the module when
+    // support for that is not available.
+    scriptDirectory = require('url').fileURLToPath(new URL('./', import.meta.url)); // includes trailing slash
+#else
     scriptDirectory = __dirname + '/';
+#endif
   }
 
 #include "node_shell_read.js"

test/test_other.py

@@ -238,41 +238,68 @@ def test_emcc_generate_config(self):
         self.assertContained('LLVM_ROOT', config_contents)
         os.remove(config_path)
 
-  def test_emcc_output_mjs(self):
-    self.run_process([EMCC, '-o', 'hello_world.mjs', test_file('hello_world.c')])
-    output = read_file('hello_world.mjs')
-    self.assertContained('export default Module;', output)
-    # TODO(sbc): Test that this is actually runnable.  We currently don't have
-    # any tests for EXPORT_ES6 but once we do this should be enabled.
-    # self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
+  @parameterized({
+    '': ([],),
+    'node': (['-sENVIRONMENT=node'],),
+  })
+  def test_emcc_output_mjs(self, args):
+    create_file('extern-post.js', 'await Module();')
+    self.run_process([EMCC, '-o', 'hello_world.mjs',
+                      '--extern-post-js', 'extern-post.js',
+                      test_file('hello_world.c')] + args)
+    src = read_file('hello_world.mjs')
+    self.assertContained('export default Module;', src)
+    self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
 
   @parameterized({
-    '': (True, [],),
-    'no_import_meta': (False, ['-sUSE_ES6_IMPORT_META=0'],),
+    '': ([],),
+    'node': (['-sENVIRONMENT=node'],),
   })
-  def test_emcc_output_worker_mjs(self, has_import_meta, args):
+  @node_pthreads
+  def test_emcc_output_worker_mjs(self, args):
+    create_file('extern-post.js', 'await Module();')
     os.mkdir('subdir')
-    self.run_process([EMCC, '-o', 'subdir/hello_world.mjs', '-pthread', '-O1',
+    self.run_process([EMCC, '-o', 'subdir/hello_world.mjs',
+                      '-sEXIT_RUNTIME', '-sPROXY_TO_PTHREAD', '-pthread', '-O1',
+                      '--extern-post-js', 'extern-post.js',
                       test_file('hello_world.c')] + args)
     src = read_file('subdir/hello_world.mjs')
-    self.assertContainedIf("new URL('hello_world.wasm', import.meta.url)", src, condition=has_import_meta)
-    self.assertContainedIf("new Worker(new URL('hello_world.worker.js', import.meta.url))", src, condition=has_import_meta)
+    self.assertContained("new URL('hello_world.wasm', import.meta.url)", src)
+    self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url))", src)
     self.assertContained('export default Module;', src)
     src = read_file('subdir/hello_world.worker.js')
     self.assertContained('import("./hello_world.mjs")', src)
+    self.assertContained('hello, world!', self.run_js('subdir/hello_world.mjs'))
 
+  @node_pthreads
   def test_emcc_output_worker_mjs_single_file(self):
+    create_file('extern-post.js', 'await Module();')
     self.run_process([EMCC, '-o', 'hello_world.mjs', '-pthread',
+                     '--extern-post-js', 'extern-post.js',
                       test_file('hello_world.c'), '-sSINGLE_FILE'])
     src = read_file('hello_world.mjs')
     self.assertNotContained("new URL('data:", src)
     self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url))", src)
+    self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
 
   def test_emcc_output_mjs_closure(self):
+    create_file('extern-post.js', 'await Module();')
     self.run_process([EMCC, '-o', 'hello_world.mjs',
+                      '--extern-post-js', 'extern-post.js',
                       test_file('hello_world.c'), '--closure=1'])
     src = read_file('hello_world.mjs')
     self.assertContained('new URL("hello_world.wasm", import.meta.url)', src)
+    self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
+
+  def test_emcc_output_mjs_web_no_import_meta(self):
+    # Ensure we don't emit import.meta.url at all for:
+    # ENVIRONMENT=web + EXPORT_ES6 + USE_ES6_IMPORT_META=0
+    self.run_process([EMCC, '-o', 'hello_world.mjs',
+                      test_file('hello_world.c'),
+                      '-sENVIRONMENT=web', '-sUSE_ES6_IMPORT_META=0'])
+    src = read_file('hello_world.mjs')
+    self.assertNotContained('import.meta.url', src)
+    self.assertContained('export default Module;', src)
 
   def test_export_es6_implies_modularize(self):
     self.run_process([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6'])
@@ -283,6 +310,11 @@ def test_export_es6_requires_modularize(self):
     err = self.expect_fail([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6', '-sMODULARIZE=0'])
     self.assertContained('EXPORT_ES6 requires MODULARIZE to be set', err)
 
+  def test_export_es6_node_requires_import_meta(self):
+    err = self.expect_fail([EMCC, test_file('hello_world.c'),
+                            '-sENVIRONMENT=node', '-sEXPORT_ES6', '-sUSE_ES6_IMPORT_META=0'])
+    self.assertContained('EXPORT_ES6 and ENVIRONMENT=*node* requires USE_ES6_IMPORT_META to be set', err)
+
   def test_export_es6_allows_export_in_post_js(self):
     self.run_process([EMCC, test_file('hello_world.c'), '-O3', '-sEXPORT_ES6', '--post-js', test_file('export_module.js')])
     src = read_file('a.out.js')

third_party/closure-compiler/node-externs/url.js

@@ -61,3 +61,15 @@ url.format = function(urlObj) {};
  * @nosideeffects
  */
 url.resolve = function(from, to) {};
+
+/**
+ * @param {url.URL|string} url
+ * @return {string}
+ */
+url.fileURLToPath = function(url) {};
+
+/**
+ * @param {string} path
+ * @return {url.URL}
+ */
+url.pathToFileURL = function(path) {};