src: support import() and import.meta in embedder-run modules

This adds a embedder_module_hdo for identifying embedder-run
modules in the dynamic import handler and import.meta initializer,
and a SourceTextModuleTypes for customizing source text module
compilation in the JS land via compileSourceTextModule().
Also, refactors the existing embedder module compilation code
to reuse the builtin resolution logic.

PR-URL: #61654
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
Reviewed-By: Aditi Singh <aditisingh1400@gmail.com>

Author: joyeecheung

lib/internal/main/embedding.js

@@ -15,16 +15,12 @@ const {
 const { isExperimentalSeaWarningNeeded, isSea } = internalBinding('sea');
 const { emitExperimentalWarning } = require('internal/util');
 const { emitWarningSync } = require('internal/process/warning');
-const { BuiltinModule } = require('internal/bootstrap/realm');
-const { normalizeRequirableId } = BuiltinModule;
 const { Module } = require('internal/modules/cjs/loader');
 const { compileFunctionForCJSLoader } = internalBinding('contextify');
 const { maybeCacheSourceMap } = require('internal/source_map/source_map_cache');
-const { codes: {
-  ERR_UNKNOWN_BUILTIN_MODULE,
-} } = require('internal/errors');
 const { pathToFileURL } = require('internal/url');
-const { loadBuiltinModule } = require('internal/modules/helpers');
+const { loadBuiltinModuleForEmbedder } = require('internal/modules/helpers');
+const { compileSourceTextModule, SourceTextModuleTypes: { kEmbedder } } = require('internal/modules/esm/utils');
 const { moduleFormats } = internalBinding('modules');
 const assert = require('internal/assert');
 const path = require('path');
@@ -34,7 +30,6 @@ const path = require('path');
 prepareMainThreadExecution(false, true);
 
 const isLoadingSea = isSea();
-const isBuiltinWarningNeeded = isLoadingSea && isExperimentalSeaWarningNeeded();
 if (isExperimentalSeaWarningNeeded()) {
   emitExperimentalWarning('Single executable application');
 }
@@ -65,6 +60,7 @@ function embedderRunCjs(content, filename) {
     filename,
     isLoadingSea, // is_sea_main
     false, // should_detect_module, ESM should be supported differently for embedded code
+    true,  // is_embedder
   );
   // Cache the source map for the module if present.
   if (sourceMapURL) {
@@ -103,28 +99,8 @@ function embedderRunCjs(content, filename) {
   );
 }
 
-let warnedAboutBuiltins = false;
-function warnNonBuiltinInSEA() {
-  if (isBuiltinWarningNeeded && !warnedAboutBuiltins) {
-    emitWarningSync(
-      'Currently the require() provided to the main script embedded into ' +
-      'single-executable applications only supports loading built-in modules.\n' +
-      'To load a module from disk after the single executable application is ' +
-      'launched, use require("module").createRequire().\n' +
-      'Support for bundled module loading or virtual file systems are under ' +
-      'discussions in https://github.com/nodejs/single-executable');
-    warnedAboutBuiltins = true;
-  }
-}
-
 function embedderRequire(id) {
-  const normalizedId = normalizeRequirableId(id);
-
-  if (!normalizedId) {
-    warnNonBuiltinInSEA();
-    throw new ERR_UNKNOWN_BUILTIN_MODULE(id);
-  }
-  return require(normalizedId);
+  return loadBuiltinModuleForEmbedder(id).exports;
 }
 
 function embedderRunESM(content, filename) {
@@ -134,31 +110,10 @@ function embedderRunESM(content, filename) {
   } else {
     resourceName = filename;
   }
-  const { compileSourceTextModule } = require('internal/modules/esm/utils');
-  // TODO(joyeecheung): support code cache, dynamic import() and import.meta.
-  const wrap = compileSourceTextModule(resourceName, content);
-  // Cache the source map for the module if present.
-  if (wrap.sourceMapURL) {
-    maybeCacheSourceMap(resourceName, content, wrap, false, undefined, wrap.sourceMapURL);
-  }
-  const requests = wrap.getModuleRequests();
-  const modules = [];
-  for (let i = 0; i < requests.length; ++i) {
-    const { specifier } = requests[i];
-    const normalizedId = normalizeRequirableId(specifier);
-    if (!normalizedId) {
-      warnNonBuiltinInSEA();
-      throw new ERR_UNKNOWN_BUILTIN_MODULE(specifier);
-    }
-    const mod = loadBuiltinModule(normalizedId);
-    if (!mod) {
-      throw new ERR_UNKNOWN_BUILTIN_MODULE(specifier);
-    }
-    modules.push(mod.getESMFacade());
-  }
-  wrap.link(modules);
-  wrap.instantiate();
-  wrap.evaluate(-1, false);
+  // TODO(joyeecheung): allow configuration from node::ModuleData,
+  // either via a more generic context object, or something like import.meta extensions.
+  const context = { isMain: true, __proto__: null };
+  const wrap = compileSourceTextModule(resourceName, content, kEmbedder, context);
 
   // TODO(joyeecheung): we may want to return the v8::Module via a vm.SourceTextModule
   // when vm.SourceTextModule stablizes, or put it in an out parameter.

lib/internal/modules/esm/create_dynamic_module.js

@@ -58,8 +58,10 @@ ${ArrayPrototypeJoin(ArrayPrototypeMap(imports, createImport), '\n')}
 ${ArrayPrototypeJoin(ArrayPrototypeMap(exports, createExport), '\n')}
 import.meta.done();
 `;
-  const { registerModule, compileSourceTextModule } = require('internal/modules/esm/utils');
-  const m = compileSourceTextModule(`${url}`, source);
+  const {
+    registerModule, compileSourceTextModule, SourceTextModuleTypes: { kFacade },
+  } = require('internal/modules/esm/utils');
+  const m = compileSourceTextModule(`${url}`, source, kFacade);
 
   const readyfns = new SafeSet();
   /** @type {DynamicModuleReflect} */

lib/internal/modules/esm/loader.js

@@ -34,6 +34,7 @@ const { isURL, pathToFileURL } = require('internal/url');
 const { kEmptyObject } = require('internal/util');
 const {
   compileSourceTextModule,
+  SourceTextModuleTypes: { kUser },
   getDefaultConditions,
   shouldSpawnLoaderHookWorker,
   requestTypes: { kImportInRequiredESM, kRequireInImportedCJS, kImportInImportedESM },
@@ -244,7 +245,7 @@ class ModuleLoader {
    * @returns {object} The module wrap object.
    */
   createModuleWrap(source, url, context = kEmptyObject) {
-    return compileSourceTextModule(url, source, this, context);
+    return compileSourceTextModule(url, source, kUser, context);
   }
 
   /**
@@ -371,7 +372,7 @@ class ModuleLoader {
     // TODO(joyeecheung): refactor this so that we pre-parse in C++ and hit the
     // cache here, or use a carrier object to carry the compiled module script
     // into the constructor to ensure cache hit.
-    const wrap = compileSourceTextModule(url, source, this);
+    const wrap = compileSourceTextModule(url, source, kUser);
     const inspectBrk = (isMain && getOptionValue('--inspect-brk'));
 
     const { ModuleJobSync } = require('internal/modules/esm/module_job');

lib/internal/modules/esm/translators.js

@@ -93,9 +93,11 @@ translators.set('module', function moduleStrategy(url, translateContext, parentU
   assertBufferSource(source, true, 'load');
   source = stringify(source);
   debug(`Translating StandardModule ${url}`, translateContext);
-  const { compileSourceTextModule } = require('internal/modules/esm/utils');
+  const {
+    compileSourceTextModule, SourceTextModuleTypes: { kUser },
+  } = require('internal/modules/esm/utils');
   const context = isMain ? { isMain } : undefined;
-  const module = compileSourceTextModule(url, source, this, context);
+  const module = compileSourceTextModule(url, source, kUser, context);
   return module;
 });
 

lib/internal/modules/esm/utils.js

@@ -14,6 +14,7 @@ const {
   },
 } = internalBinding('util');
 const {
+  embedder_module_hdo,
   source_text_module_default_hdo,
   vm_dynamic_import_default_internal,
   vm_dynamic_import_main_context_default,
@@ -43,6 +44,7 @@ const {
 const assert = require('internal/assert');
 const {
   normalizeReferrerURL,
+  loadBuiltinModuleForEmbedder,
 } = require('internal/modules/helpers');
 
 let defaultConditions;
@@ -226,6 +228,28 @@ function defaultImportModuleDynamicallyForScript(specifier, phase, attributes, r
   return cascadedLoader.import(specifier, parentURL, attributes, phase);
 }
 
+/**
+ * Loads the built-in and wraps it in a ModuleWrap for embedder ESM.
+ * @param {string} specifier
+ * @returns {ModuleWrap}
+ */
+function getBuiltinModuleWrapForEmbedder(specifier) {
+  return loadBuiltinModuleForEmbedder(specifier).getESMFacade();
+}
+
+/**
+ * Get the built-in module dynamically for embedder ESM.
+ * @param {string} specifier - The module specifier string.
+ * @param {number} phase - The module import phase. Ignored for now.
+ * @param {Record<string, string>} attributes - The import attributes object. Ignored for now.
+ * @param {string|null|undefined} referrerName - name of the referrer.
+ * @returns {import('internal/modules/esm/loader.js').ModuleExports} - The imported module object.
+ */
+function importModuleDynamicallyForEmbedder(specifier, phase, attributes, referrerName) {
+  // Ignore phase and attributes for embedder ESM for now, because this only supports loading builtins.
+  return getBuiltinModuleWrapForEmbedder(specifier).getNamespace();
+}
+
 /**
  * Asynchronously imports a module dynamically using a callback function. The native callback.
  * @param {symbol} referrerSymbol - Referrer symbol of the registered script, function, module, or contextified object.
@@ -253,6 +277,10 @@ async function importModuleDynamicallyCallback(referrerSymbol, specifier, phase,
   if (referrerSymbol === source_text_module_default_hdo) {
     return defaultImportModuleDynamicallyForModule(specifier, phase, attributes, referrerName);
   }
+  // For embedder entry point ESM, only allow built-in modules.
+  if (referrerSymbol === embedder_module_hdo) {
+    return importModuleDynamicallyForEmbedder(specifier, phase, attributes, referrerName);
+  }
 
   if (moduleRegistries.has(referrerSymbol)) {
     const { importModuleDynamically, callbackReferrer } = moduleRegistries.get(referrerSymbol);
@@ -290,21 +318,42 @@ function shouldSpawnLoaderHookWorker() {
   return _shouldSpawnLoaderHookWorker;
 }
 
+const SourceTextModuleTypes = {
+  kInternal: 'internal',  // TODO(joyeecheung): support internal ESM.
+  kEmbedder: 'embedder',  // Embedder ESM, also used by SEA
+  kUser: 'user', // User-land ESM
+  kFacade: 'facade',  // Currently only used by the facade that proxies WASM module import/exports.
+};
+
 /**
  * Compile a SourceTextModule for the built-in ESM loader. Register it for default
  * source map and import.meta and dynamic import() handling if cascadedLoader is provided.
  * @param {string} url URL of the module.
  * @param {string} source Source code of the module.
- * @param {typeof import('./loader.js').ModuleLoader|undefined} cascadedLoader If provided,
- *   register the module for default handling.
+ * @param {string} type Type of the source text module, one of SourceTextModuleTypes.
  * @param {{ isMain?: boolean }|undefined} context - context object containing module metadata.
  * @returns {ModuleWrap}
  */
-function compileSourceTextModule(url, source, cascadedLoader, context = kEmptyObject) {
-  const hostDefinedOption = cascadedLoader ? source_text_module_default_hdo : undefined;
-  const wrap = new ModuleWrap(url, undefined, source, 0, 0, hostDefinedOption);
+function compileSourceTextModule(url, source, type, context = kEmptyObject) {
+  let hostDefinedOptions;
+  switch (type) {
+    case SourceTextModuleTypes.kFacade:
+    case SourceTextModuleTypes.kInternal:
+      hostDefinedOptions = undefined;
+      break;
+    case SourceTextModuleTypes.kEmbedder:
+      hostDefinedOptions = embedder_module_hdo;
+      break;
+    case SourceTextModuleTypes.kUser:
+      hostDefinedOptions = source_text_module_default_hdo;
+      break;
+    default:
+      assert.fail(`Unknown SourceTextModule type: ${type}`);
+  }
+
+  const wrap = new ModuleWrap(url, undefined, source, 0, 0, hostDefinedOptions);
 
-  if (!cascadedLoader) {
+  if (type === SourceTextModuleTypes.kFacade) {
     return wrap;
   }
 
@@ -317,10 +366,18 @@ function compileSourceTextModule(url, source, cascadedLoader, context = kEmptyOb
   if (wrap.sourceMapURL) {
     maybeCacheSourceMap(url, source, wrap, false, wrap.sourceURL, wrap.sourceMapURL);
   }
+
+  if (type === SourceTextModuleTypes.kEmbedder) {
+    // For embedder ESM, we also handle the linking and evaluation.
+    const requests = wrap.getModuleRequests();
+    const modules = requests.map(({ specifier }) => getBuiltinModuleWrapForEmbedder(specifier));
+    wrap.link(modules);
+    wrap.instantiate();
+    wrap.evaluate(-1, false);
+  }
   return wrap;
 }
 
-
 const kImportInImportedESM = Symbol('kImportInImportedESM');
 const kImportInRequiredESM = Symbol('kImportInRequiredESM');
 const kRequireInImportedCJS = Symbol('kRequireInImportedCJS');
@@ -331,11 +388,13 @@ const kRequireInImportedCJS = Symbol('kRequireInImportedCJS');
 const requestTypes = { kImportInImportedESM, kImportInRequiredESM, kRequireInImportedCJS };
 
 module.exports = {
+  embedder_module_hdo,
   registerModule,
   initializeESM,
   getDefaultConditions,
   getConditionsSet,
   shouldSpawnLoaderHookWorker,
   compileSourceTextModule,
+  SourceTextModuleTypes,
   requestTypes,
 };

lib/internal/modules/helpers.js

@@ -15,6 +15,7 @@ const {
 const {
   ERR_INVALID_ARG_TYPE,
   ERR_INVALID_RETURN_PROPERTY_VALUE,
+  ERR_UNKNOWN_BUILTIN_MODULE,
 } = require('internal/errors').codes;
 const { BuiltinModule } = require('internal/bootstrap/realm');
 
@@ -28,6 +29,7 @@ const assert = require('internal/assert');
 const { getOptionValue } = require('internal/options');
 const { setOwnProperty, getLazy } = require('internal/util');
 const { inspect } = require('internal/util/inspect');
+const { emitWarningSync } = require('internal/process/warning');
 
 const lazyTmpdir = getLazy(() => require('os').tmpdir());
 const { join } = path;
@@ -126,6 +128,42 @@ function loadBuiltinModule(id) {
   return mod;
 }
 
+let isSEABuiltinWarningNeeded_;
+function isSEABuiltinWarningNeeded() {
+  if (isSEABuiltinWarningNeeded_ === undefined) {
+    const { isExperimentalSeaWarningNeeded, isSea } = internalBinding('sea');
+    isSEABuiltinWarningNeeded_ = isSea() && isExperimentalSeaWarningNeeded();
+  }
+  return isSEABuiltinWarningNeeded_;
+}
+
+let warnedAboutBuiltins = false;
+/**
+ * Helpers to load built-in modules for embedder modules.
+ * @param {string} id
+ * @returns {import('internal/bootstrap/realm.js').BuiltinModule}
+ */
+function loadBuiltinModuleForEmbedder(id) {
+  const normalized = BuiltinModule.normalizeRequirableId(id);
+  if (normalized) {
+    const mod = loadBuiltinModule(normalized);
+    if (mod) {
+      return mod;
+    }
+  }
+  if (isSEABuiltinWarningNeeded() && !warnedAboutBuiltins) {
+    emitWarningSync(
+      'Currently the require() provided to the main script embedded into ' +
+      'single-executable applications only supports loading built-in modules.\n' +
+      'To load a module from disk after the single executable application is ' +
+      'launched, use require("module").createRequire().\n' +
+      'Support for bundled module loading or virtual file systems are under ' +
+      'discussions in https://github.com/nodejs/single-executable');
+    warnedAboutBuiltins = true;
+  }
+  throw new ERR_UNKNOWN_BUILTIN_MODULE(id);
+}
+
 /** @type {Module} */
 let $Module = null;
 /**
@@ -469,6 +507,7 @@ module.exports = {
   getCjsConditionsArray,
   getCompileCacheDir,
   initializeCjsConditions,
+  loadBuiltinModuleForEmbedder,
   loadBuiltinModule,
   makeRequireFunction,
   normalizeReferrerURL,

src/env_properties.h

@@ -58,6 +58,7 @@
   V(resource_symbol, "resource_symbol")                                        \
   V(trigger_async_id_symbol, "trigger_async_id_symbol")                        \
   V(builtin_source_text_module_hdo, "builtin_source_text_module_hdo")          \
+  V(embedder_module_hdo, "embedder_module_hdo")                                \
   V(source_text_module_default_hdo, "source_text_module_default_hdo")          \
   V(vm_context_no_contextify, "vm_context_no_contextify")                      \
   V(vm_dynamic_import_default_internal, "vm_dynamic_import_default_internal")  \

src/module_wrap.cc

@@ -1397,7 +1397,8 @@ void ModuleWrap::HostInitializeImportMetaObjectCallback(
 
   // Use the default initializer for source text modules without custom
   // callbacks.
-  if (id == env->source_text_module_default_hdo()) {
+  if (id == env->source_text_module_default_hdo() ||
+      id == env->embedder_module_hdo()) {
     USE(DefaultImportMetaObjectInitializer(realm, wrap, meta));
     return;
   }