vm: support using the default loader to handle dynamic import()

This patch adds support for using
`vm.constants.USE_MAIN_CONTEXT_DEFAULT_LOADER` as
`importModuleDynamically` in all APIs that take the option
except `vm.SourceTextModule`. This allows users to have a shortcut
to support dynamic import() in the compiled code without missing
the compilation cache if they don't need customization of the
loading process. We emit an experimental warning when the
`import()` is actually handled by the default loader through
this option instead of requiring `--experimental-vm-modules`.

In addition this refactors the documentation for
`importModuleDynamically` and adds a dedicated section for it
with examples.

`vm.SourceTextModule` is not supported in this patch because
it needs additional refactoring to handle `initializeImportMeta`,
which can be done in a follow-up.

lib/internal/modules/cjs/loader.js

@@ -52,7 +52,6 @@ const {
   SafeMap,
   SafeWeakMap,
   String,
-  Symbol,
   StringPrototypeCharAt,
   StringPrototypeCharCodeAt,
   StringPrototypeEndsWith,
@@ -107,7 +106,6 @@ const {
   initializeCjsConditions,
   loadBuiltinModule,
   makeRequireFunction,
-  normalizeReferrerURL,
   stripBOM,
   toRealPath,
 } = require('internal/modules/helpers');
@@ -121,9 +119,10 @@ const shouldReportRequiredModules = getLazy(() => process.env.WATCH_REPORT_DEPEN
 const getCascadedLoader = getLazy(
   () => require('internal/process/esm_loader').esmLoader,
 );
-
 const permission = require('internal/process/permission');
-
+const {
+  vm_dynamic_import_default_internal,
+} = internalBinding('symbols');
 // Whether any user-provided CJS modules had been loaded (executed).
 // Used for internal assertions.
 let hasLoadedAnyUserCJSModule = false;
@@ -1254,12 +1253,8 @@ let hasPausedEntry = false;
  * @param {object} codeCache The SEA code cache
  */
 function wrapSafe(filename, content, cjsModuleInstance, codeCache) {
-  const hostDefinedOptionId = Symbol(`cjs:${filename}`);
-  async function importModuleDynamically(specifier, _, importAttributes) {
-    const cascadedLoader = getCascadedLoader();
-    return cascadedLoader.import(specifier, normalizeReferrerURL(filename),
-                                 importAttributes);
-  }
+  const hostDefinedOptionId = vm_dynamic_import_default_internal;
+  const importModuleDynamically = vm_dynamic_import_default_internal;
   if (patched) {
     const wrapped = Module.wrap(content);
     const script = makeContextifyScript(

lib/internal/modules/esm/translators.js

@@ -15,7 +15,6 @@ const {
   StringPrototypeReplaceAll,
   StringPrototypeSlice,
   StringPrototypeStartsWith,
-  Symbol,
   SyntaxErrorPrototype,
   globalThis: { WebAssembly },
 } = primordials;
@@ -58,7 +57,9 @@ const { ModuleWrap } = moduleWrap;
 const asyncESM = require('internal/process/esm_loader');
 const { emitWarningSync } = require('internal/process/warning');
 const { internalCompileFunction } = require('internal/vm');
-
+const {
+  vm_dynamic_import_default_internal,
+} = internalBinding('symbols');
 // Lazy-loading to avoid circular dependencies.
 let getSourceSync;
 /**
@@ -205,9 +206,8 @@ function enrichCJSError(err, content, filename) {
  */
 function loadCJSModule(module, source, url, filename) {
   let compiledWrapper;
-  async function importModuleDynamically(specifier, _, importAttributes) {
-    return asyncESM.esmLoader.import(specifier, url, importAttributes);
-  }
+  const hostDefinedOptionId = vm_dynamic_import_default_internal;
+  const importModuleDynamically = vm_dynamic_import_default_internal;
   try {
     compiledWrapper = internalCompileFunction(
       source,                         // code,
@@ -225,8 +225,8 @@ function loadCJSModule(module, source, url, filename) {
         '__filename',
         '__dirname',
       ],
-      Symbol(`cjs:${filename}`),      // hostDefinedOptionsId
-      importModuleDynamically,        // importModuleDynamically
+      hostDefinedOptionId,           // hostDefinedOptionsId
+      importModuleDynamically,       // importModuleDynamically
     ).function;
   } catch (err) {
     enrichCJSError(err, source, filename);

lib/internal/modules/esm/utils.js

@@ -14,8 +14,10 @@ const {
   },
 } = internalBinding('util');
 const {
-  default_host_defined_options,
+  vm_dynamic_import_default_internal,
+  vm_dynamic_import_main_context_default,
   vm_dynamic_import_missing_flag,
+  vm_dynamic_import_no_callback,
 } = internalBinding('symbols');
 
 const {
@@ -28,12 +30,19 @@ const {
   loadPreloadModules,
   initializeFrozenIntrinsics,
 } = require('internal/process/pre_execution');
-const { getCWDURL } = require('internal/util');
+const {
+  emitExperimentalWarning,
+  getCWDURL,
+  getLazy,
+} = require('internal/util');
 const {
   setImportModuleDynamicallyCallback,
   setInitializeImportMetaObjectCallback,
 } = internalBinding('module_wrap');
 const assert = require('internal/assert');
+const {
+  normalizeReferrerURL,
+} = require('internal/modules/helpers');
 
 let defaultConditions;
 /**
@@ -145,8 +154,10 @@ const moduleRegistries = new SafeWeakMap();
  */
 function registerModule(referrer, registry) {
   const idSymbol = referrer[host_defined_option_symbol];
-  if (idSymbol === default_host_defined_options ||
-      idSymbol === vm_dynamic_import_missing_flag) {
+  if (idSymbol === vm_dynamic_import_no_callback ||
+      idSymbol === vm_dynamic_import_missing_flag ||
+      idSymbol === vm_dynamic_import_main_context_default ||
+      idSymbol === vm_dynamic_import_default_internal) {
     // The referrer is compiled without custom callbacks, so there is
     // no registry to hold on to. We'll throw
     // ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING when a callback is
@@ -191,16 +202,36 @@ function initializeImportMetaObject(symbol, meta) {
     }
   }
 }
+const getCascadedLoader = getLazy(
+  () => require('internal/process/esm_loader').esmLoader,
+);
+function defaultImportModuleDynamically(specifier, attributes, referrerName) {
+  const parentURL = normalizeReferrerURL(referrerName);
+  return getCascadedLoader().import(specifier, parentURL, attributes);
+}
 
 /**
  * Asynchronously imports a module dynamically using a callback function. The native callback.
  * @param {symbol} referrerSymbol - Referrer symbol of the registered script, function, module, or contextified object.
  * @param {string} specifier - The module specifier string.
  * @param {Record<string, string>} attributes - The import attributes object.
+ * @param {string|null} referrerName - name of the referrer.
  * @returns {Promise<import('internal/modules/esm/loader.js').ModuleExports>} - The imported module object.
  * @throws {ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING} - If the callback function is missing.
  */
-async function importModuleDynamicallyCallback(referrerSymbol, specifier, attributes) {
+async function importModuleDynamicallyCallback(referrerSymbol, specifier, attributes, referrerName) {
+  // For user-provided vm.constants.USE_MAIN_CONTEXT_DEFAULT_LOADER, emit the warning
+  // and fall back to the default loader.
+  if (referrerSymbol === vm_dynamic_import_main_context_default) {
+    emitExperimentalWarning('vm.USE_MAIN_CONTEXT_DEFAULT_LOADER');
+    return defaultImportModuleDynamically(specifier, attributes, referrerName);
+  }
+  // For script compiled internally that should use the default loader to handle dynamic
+  // import, proxy the request to the default loader without the warning.
+  if (referrerSymbol === vm_dynamic_import_default_internal) {
+    return defaultImportModuleDynamically(specifier, attributes, referrerName);
+  }
+
   if (moduleRegistries.has(referrerSymbol)) {
     const { importModuleDynamically, callbackReferrer } = moduleRegistries.get(referrerSymbol);
     if (importModuleDynamically !== undefined) {

lib/internal/modules/helpers.js

@@ -33,6 +33,7 @@ const {
     require_private_symbol,
   },
 } = internalBinding('util');
+const { canParse: URLCanParse } = internalBinding('url');
 
 let debug = require('internal/util/debuglog').debuglog('module', (fn) => {
   debug = fn;
@@ -288,14 +289,30 @@ function addBuiltinLibsToObject(object, dummyModuleName) {
 }
 
 /**
- * If a referrer is an URL instance or absolute path, convert it into an URL string.
- * @param {string | URL} referrer
+ * Normalize the referrer name as a URL.
+ * If it's an absolute path or a file:// it's normalized as a file:// URL.
+ * Otherwise it's returned as undefined;
+ * @param {string | null | undefined | false | any } referrerName
+ * @returns {string | undefined}
  */
-function normalizeReferrerURL(referrer) {
-  if (typeof referrer === 'string' && path.isAbsolute(referrer)) {
-    return pathToFileURL(referrer).href;
+function normalizeReferrerURL(referrerName) {
+  if (typeof referrerName !== 'string') {
+    return undefined;
   }
-  return new URL(referrer).href;
+
+  if (StringPrototypeStartsWith(referrerName, 'file://')) {
+    return referrerName;
+  }
+
+  if (path.isAbsolute(referrerName)) {
+    return pathToFileURL(referrerName).href;
+  }
+
+  if (URLCanParse(referrerName)) {
+    return new URL(referrerName).href;
+  }
+
+  return undefined;
 }
 
 module.exports = {

lib/internal/process/pre_execution.js

@@ -76,15 +76,21 @@ function prepareShadowRealmExecution() {
 
   // Disable custom loaders in ShadowRealm.
   setupUserModules(true);
-  registerRealm(globalThis, {
-    __proto__: null,
-    importModuleDynamically: (specifier, _referrer, attributes) => {
-      // The handler for `ShadowRealm.prototype.importValue`.
-      const { esmLoader } = require('internal/process/esm_loader');
-      // `parentURL` is not set in the case of a ShadowRealm top-level import.
-      return esmLoader.import(specifier, undefined, attributes);
+  const {
+    privateSymbols: {
+      host_defined_option_symbol,
     },
-  });
+  } = internalBinding('util');
+  const {
+    vm_dynamic_import_default_internal,
+  } = internalBinding('symbols');
+
+  // For ShadowRealm.prototype.importValue(), the referrer name is
+  // always null which would be coerced to an undefined parentURL.
+  // when we use vm_dynamic_import_default_internal
+  // to proxy the request to the default handler.
+  globalThis[host_defined_option_symbol] =
+    vm_dynamic_import_default_internal;
 }
 
 function prepareExecution(options) {

lib/internal/source_map/source_map_cache.js

@@ -105,12 +105,10 @@ function extractSourceMapURLMagicComment(content) {
 function maybeCacheSourceMap(filename, content, cjsModuleInstance, isGeneratedSource, sourceURL, sourceMapURL) {
   const sourceMapsEnabled = getSourceMapsEnabled();
   if (!(process.env.NODE_V8_COVERAGE || sourceMapsEnabled)) return;
-  try {
-    const { normalizeReferrerURL } = require('internal/modules/helpers');
-    filename = normalizeReferrerURL(filename);
-  } catch (err) {
+  const { normalizeReferrerURL } = require('internal/modules/helpers');
+  filename = normalizeReferrerURL(filename);
+  if (filename === undefined) {
     // This is most likely an invalid filename in sourceURL of [eval]-wrapper.
-    debug(err);
     return;
   }
 

lib/internal/vm.js

@@ -14,7 +14,9 @@ const {
   runInContext,
 } = ContextifyScript.prototype;
 const {
-  default_host_defined_options,
+  vm_dynamic_import_default_internal,
+  vm_dynamic_import_main_context_default,
+  vm_dynamic_import_no_callback,
   vm_dynamic_import_missing_flag,
 } = internalBinding('symbols');
 const {
@@ -27,14 +29,18 @@ const {
   getOptionValue,
 } = require('internal/options');
 
-
 function isContext(object) {
   validateObject(object, 'object', kValidateObjectAllowArray);
 
   return _isContext(object);
 }
 
 function getHostDefinedOptionId(importModuleDynamically, hint) {
+  if (importModuleDynamically === vm_dynamic_import_main_context_default ||
+      importModuleDynamically === vm_dynamic_import_default_internal) {
+    return importModuleDynamically;
+  }
+
   if (importModuleDynamically !== undefined) {
     // Check that it's either undefined or a function before we pass
     // it into the native constructor.
@@ -45,7 +51,7 @@ function getHostDefinedOptionId(importModuleDynamically, hint) {
     // We need a default host defined options that are the same for all
     // scripts not needing custom module callbacks so that the isolate
     // compilation cache can be hit.
-    return default_host_defined_options;
+    return vm_dynamic_import_no_callback;
   }
   // We should've thrown here immediately when we introduced
   // --experimental-vm-modules and importModuleDynamically, but since
@@ -61,6 +67,13 @@ function getHostDefinedOptionId(importModuleDynamically, hint) {
 }
 
 function registerImportModuleDynamically(referrer, importModuleDynamically) {
+  // If it's undefined or certain known symbol, there's no customization so
+  // no need to register anything.
+  if (importModuleDynamically === undefined ||
+      importModuleDynamically === vm_dynamic_import_main_context_default ||
+      importModuleDynamically === vm_dynamic_import_default_internal) {
+    return;
+  }
   const { importModuleDynamicallyWrap } = require('internal/vm/module');
   const { registerModule } = require('internal/modules/esm/utils');
   registerModule(referrer, {
@@ -99,9 +112,7 @@ function internalCompileFunction(
     result.function.cachedDataRejected = result.cachedDataRejected;
   }
 
-  if (importModuleDynamically !== undefined) {
-    registerImportModuleDynamically(result.function, importModuleDynamically);
-  }
+  registerImportModuleDynamically(result.function, importModuleDynamically);
 
   return result;
 }
@@ -132,9 +143,7 @@ function makeContextifyScript(code,
     throw e; /* node-do-not-add-exception-line */
   }
 
-  if (importModuleDynamically !== undefined) {
-    registerImportModuleDynamically(script, importModuleDynamically);
-  }
+  registerImportModuleDynamically(script, importModuleDynamically);
   return script;
 }
 

lib/vm.js

@@ -23,6 +23,7 @@
 
 const {
   ArrayPrototypeForEach,
+  ObjectFreeze,
   Symbol,
   PromiseReject,
   ReflectApply,
@@ -61,6 +62,9 @@ const {
   isContext,
   registerImportModuleDynamically,
 } = require('internal/vm');
+const {
+  vm_dynamic_import_main_context_default,
+} = internalBinding('symbols');
 const kParsingContext = Symbol('script parsing context');
 
 class Script extends ContextifyScript {
@@ -108,9 +112,7 @@ class Script extends ContextifyScript {
       throw e; /* node-do-not-add-exception-line */
     }
 
-    if (importModuleDynamically !== undefined) {
-      registerImportModuleDynamically(this, importModuleDynamically);
-    }
+    registerImportModuleDynamically(this, importModuleDynamically);
   }
 
   runInThisContext(options) {
@@ -245,9 +247,7 @@ function createContext(contextObject = {}, options = kEmptyObject) {
 
   makeContext(contextObject, name, origin, strings, wasm, microtaskQueue, hostDefinedOptionId);
   // Register the context scope callback after the context was initialized.
-  if (importModuleDynamically !== undefined) {
-    registerImportModuleDynamically(contextObject, importModuleDynamically);
-  }
+  registerImportModuleDynamically(contextObject, importModuleDynamically);
   return contextObject;
 }
 
@@ -378,6 +378,13 @@ function measureMemory(options = kEmptyObject) {
   return result;
 }
 
+const vmConstants = {
+  __proto__: null,
+  USE_MAIN_CONTEXT_DEFAULT_LOADER: vm_dynamic_import_main_context_default,
+};
+
+ObjectFreeze(vmConstants);
+
 module.exports = {
   Script,
   createContext,
@@ -388,6 +395,7 @@ module.exports = {
   isContext,
   compileFunction,
   measureMemory,
+  constants: vmConstants,
 };
 
 // The vm module is patched to include vm.Module, vm.SourceTextModule