vm: add dynamic import callback and streamline initialize import.meta callback to match it

doc/api/errors.md

@@ -1036,6 +1036,12 @@ is set for the `Http2Stream`.
 `http2.connect()` was passed a URL that uses any protocol other than `http:` or
 `https:`.
 
+<a id="ERR_DYNAMIC_IMPORT_CALLBACK_MISSING"></a>
+### ERR_DYNAMIC_IMPORT_CALLBACK_MISSING
+
+There was a dynamic import request but the script or module did not provide
+a callback to handle it.
+
 <a id="ERR_INDEX_OUT_OF_RANGE"></a>
 ### ERR_INDEX_OUT_OF_RANGE
 

doc/api/vm.md

@@ -437,6 +437,9 @@ changes:
     `cachedData` property of the returned `vm.Script` instance.
     The `cachedDataProduced` value will be set to either `true` or `false`
     depending on whether code cache data is produced successfully.
+  * `resolveDynamicImport` {Function} See [`module.link()`][]
+    * `specifier` {string}
+    * `scriptOrModule` {vm.Script|vm.Module|Module Namespace Object}
 
 Creating a new `vm.Script` object compiles `code` but does not run it. The
 compiled `vm.Script` can be run later multiple times. The `code` is not bound to
@@ -894,6 +897,7 @@ associating it with the `sandbox` object is what this document refers to as
 [`Error`]: errors.html#errors_class_error
 [`URL`]: url.html#url_class_url
 [`eval()`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/eval
+[`module.link()`]: #vm_module_link_linker
 [`script.runInContext()`]: #vm_script_runincontext_contextifiedsandbox_options
 [`script.runInThisContext()`]: #vm_script_runinthiscontext_options
 [`url.origin`]: url.html#url_url_origin

lib/internal/errors.js

@@ -927,6 +927,8 @@ E('ERR_IPC_CHANNEL_CLOSED', 'Channel closed', Error);
 E('ERR_IPC_DISCONNECTED', 'IPC channel is already disconnected', Error);
 E('ERR_IPC_ONE_PIPE', 'Child process can have only one IPC pipe', Error);
 E('ERR_IPC_SYNC_FORK', 'IPC cannot be used with synchronous forks', Error);
+E('ERR_DYNAMIC_IMPORT_CALLBACK_MISSING',
+  'No callback available for dynamic import request: %s', Error);
 E('ERR_METHOD_NOT_IMPLEMENTED', 'The %s method is not implemented', Error);
 E('ERR_MISSING_ARGS', missingArgs, TypeError);
 E('ERR_MISSING_MODULE', 'Cannot find module %s', Error);

lib/internal/modules/cjs/loader.js

@@ -646,7 +646,12 @@ Module.prototype._compile = function(content, filename) {
   var compiledWrapper = vm.runInThisContext(wrapper, {
     filename: filename,
     lineOffset: 0,
-    displayErrors: true
+    displayErrors: true,
+    resolveDynamicImport: experimentalModules ? async (specifier, script) => {
+      const loader = await asyncESM.loaderPromise;
+      const referrer = getURLFromFilePath(script.filename).href;
+      return loader.import(specifier, referrer);
+    } : undefined,
   });
 
   var inspectorWrapper = null;

lib/internal/modules/esm/translators.js

@@ -14,6 +14,11 @@ const fs = require('fs');
 const { _makeLong } = require('path');
 const { SafeMap } = require('internal/safe_globals');
 const { URL } = require('url');
+const {
+  initializeImportMetaMap,
+  importModuleDynamicallyMap,
+  loaderPromise,
+} = require('internal/process/esm_loader');
 const debug = require('util').debuglog('esm');
 const readFileAsync = require('util').promisify(fs.readFile);
 const readFileSync = fs.readFileSync;
@@ -27,10 +32,13 @@ module.exports = translators;
 translators.set('esm', async (url) => {
   const source = `${await readFileAsync(new URL(url))}`;
   debug(`Translating StandardModule ${url}`);
-  return {
-    module: new ModuleWrap(stripShebang(source), url),
-    reflect: undefined
-  };
+  const module = new ModuleWrap(stripShebang(source), url);
+  initializeImportMetaMap.set(module, (meta) => { meta.url = url; });
+  importModuleDynamicallyMap.set(module, async (specifier) => {
+    const loader = await loaderPromise;
+    return loader.import(specifier, url);
+  });
+  return { module, reflect: undefined };
 });
 
 // Strategy for loading a node-style CommonJS module

lib/internal/process/esm_loader.js

@@ -5,37 +5,11 @@ const {
   setImportModuleDynamicallyCallback,
   setInitializeImportMetaObjectCallback
 } = internalBinding('module_wrap');
-
+const {
+  ERR_DYNAMIC_IMPORT_CALLBACK_MISSING,
+} = require('internal/errors').codes;
 const { getURLFromFilePath } = require('internal/url');
 const Loader = require('internal/modules/esm/loader');
-const path = require('path');
-const { URL } = require('url');
-const {
-  initImportMetaMap,
-  wrapToModuleMap
-} = require('internal/vm/module');
-
-function normalizeReferrerURL(referrer) {
-  if (typeof referrer === 'string' && path.isAbsolute(referrer)) {
-    return getURLFromFilePath(referrer).href;
-  }
-  return new URL(referrer).href;
-}
-
-function initializeImportMetaObject(wrap, meta) {
-  const vmModule = wrapToModuleMap.get(wrap);
-  if (vmModule === undefined) {
-    // This ModuleWrap belongs to the Loader.
-    meta.url = wrap.url;
-  } else {
-    const initializeImportMeta = initImportMetaMap.get(vmModule);
-    if (initializeImportMeta !== undefined) {
-      // This ModuleWrap belongs to vm.Module, initializer callback was
-      // provided.
-      initializeImportMeta(meta, vmModule);
-    }
-  }
-}
 
 let loaderResolve;
 exports.loaderPromise = new Promise((resolve, reject) => {
@@ -45,8 +19,6 @@ exports.loaderPromise = new Promise((resolve, reject) => {
 exports.ESMLoader = undefined;
 
 exports.setup = function() {
-  setInitializeImportMetaObjectCallback(initializeImportMetaObject);
-
   let ESMLoader = new Loader();
   const loaderPromise = (async () => {
     const userLoader = process.binding('config').userLoader;
@@ -61,10 +33,24 @@ exports.setup = function() {
   })();
   loaderResolve(loaderPromise);
 
-  setImportModuleDynamicallyCallback(async (referrer, specifier) => {
-    const loader = await loaderPromise;
-    return loader.import(specifier, normalizeReferrerURL(referrer));
-  });
-
   exports.ESMLoader = ESMLoader;
 };
+
+const initializeImportMetaMap = exports.initializeImportMetaMap = new WeakMap();
+const importModuleDynamicallyMap =
+  exports.importModuleDynamicallyMap = new WeakMap();
+
+const config = process.binding('config');
+if (config.experimentalModules || config.experimentalVMModules) {
+  setInitializeImportMetaObjectCallback((meta, wrap) => {
+    if (initializeImportMetaMap.has(wrap))
+      return initializeImportMetaMap.get(wrap)(meta);
+  });
+
+  setImportModuleDynamicallyCallback(async (referrer, specifier, wrap) => {
+    if (importModuleDynamicallyMap.has(wrap))
+      return importModuleDynamicallyMap.get(wrap)(specifier);
+
+    throw new ERR_DYNAMIC_IMPORT_CALLBACK_MISSING(specifier);
+  });
+}

lib/internal/vm/module.js

@@ -12,13 +12,14 @@ const {
   ERR_VM_MODULE_LINKING_ERRORED,
   ERR_VM_MODULE_NOT_LINKED,
   ERR_VM_MODULE_NOT_MODULE,
-  ERR_VM_MODULE_STATUS
+  ERR_VM_MODULE_STATUS,
 } = require('internal/errors').codes;
 const {
   getConstructorOf,
   customInspectSymbol,
 } = require('internal/util');
 const { SafePromise } = require('internal/safe_globals');
+const { isModuleNamespaceObject } = require('util').types;
 
 const {
   ModuleWrap,
@@ -44,10 +45,7 @@ const perContextModuleId = new WeakMap();
 const wrapMap = new WeakMap();
 const dependencyCacheMap = new WeakMap();
 const linkingStatusMap = new WeakMap();
-// vm.Module -> function
-const initImportMetaMap = new WeakMap();
-// ModuleWrap -> vm.Module
-const wrapToModuleMap = new WeakMap();
+const linkerFnMap = new WeakMap();
 
 class Module {
   constructor(src, options = {}) {
@@ -62,7 +60,6 @@ class Module {
       context,
       lineOffset = 0,
       columnOffset = 0,
-      initializeImportMeta
     } = options;
 
     if (context !== undefined) {
@@ -95,19 +92,34 @@ class Module {
     validateInteger(lineOffset, 'options.lineOffset');
     validateInteger(columnOffset, 'options.columnOffset');
 
+    let { initializeImportMeta } = options;
     if (initializeImportMeta !== undefined) {
       if (typeof initializeImportMeta === 'function') {
-        initImportMetaMap.set(this, initializeImportMeta);
+        const fn = initializeImportMeta;
+        initializeImportMeta = (meta) => fn(meta, this);
       } else {
         throw new ERR_INVALID_ARG_TYPE(
           'options.initializeImportMeta', 'function', initializeImportMeta);
       }
     }
 
     const wrap = new ModuleWrap(src, url, context, lineOffset, columnOffset);
+
+    const {
+      initializeImportMetaMap,
+      importModuleDynamicallyMap,
+    } = require('internal/process/esm_loader');
+
+    if (initializeImportMeta)
+      initializeImportMetaMap.set(wrap, initializeImportMeta);
+
+    importModuleDynamicallyMap.set(wrap, async (specifier) => {
+      const linker = linkerFnMap.get(this);
+      return callLinkerForNamespace(linker, specifier, this);
+    });
+
     wrapMap.set(this, wrap);
     linkingStatusMap.set(this, 'unlinked');
-    wrapToModuleMap.set(wrap, this);
 
     Object.defineProperties(this, {
       url: { value: url, enumerable: true },
@@ -160,20 +172,9 @@ class Module {
       throw new ERR_VM_MODULE_STATUS('must be uninstantiated');
 
     linkingStatusMap.set(this, 'linking');
+    linkerFnMap.set(this, linker);
 
-    const promises = wrap.link(async (specifier) => {
-      const m = await linker(specifier, this);
-      if (!m || !wrapMap.has(m))
-        throw new ERR_VM_MODULE_NOT_MODULE();
-      if (m.context !== this.context)
-        throw new ERR_VM_MODULE_DIFFERENT_CONTEXT();
-      const childLinkingStatus = linkingStatusMap.get(m);
-      if (childLinkingStatus === 'errored')
-        throw new ERR_VM_MODULE_LINKING_ERRORED();
-      if (childLinkingStatus === 'unlinked')
-        await m.link(linker);
-      return wrapMap.get(m);
-    });
+    const promises = wrap.link((s) => callLinkerForModuleWrap(linker, s, this));
 
     try {
       if (promises !== undefined)
@@ -252,8 +253,43 @@ function validateInteger(prop, propName) {
   }
 }
 
+async function getWrapFromModule(m, scriptOrModule, linker) {
+  if (!m || !wrapMap.has(m))
+    throw new ERR_VM_MODULE_NOT_MODULE();
+
+  if (scriptOrModule instanceof Module &&
+      (m.context !== scriptOrModule.context))
+    throw new ERR_VM_MODULE_DIFFERENT_CONTEXT();
+
+  const childLinkingStatus = linkingStatusMap.get(m);
+
+  if (childLinkingStatus === 'errored')
+    throw new ERR_VM_MODULE_LINKING_ERRORED();
+  if (childLinkingStatus === 'unlinked')
+    await m.link(linker);
+
+  return wrapMap.get(m);
+}
+
+async function callLinkerForModuleWrap(linker, specifier, scriptOrModule) {
+  const m = await linker(specifier, scriptOrModule);
+  return getWrapFromModule(m, scriptOrModule, linker);
+}
+
+async function callLinkerForNamespace(linker, specifier, scriptOrModule) {
+  const m = await linker(specifier, scriptOrModule);
+  if (isModuleNamespaceObject(m))
+    return m;
+  const wrap = await getWrapFromModule(m, scriptOrModule, linker);
+  const status = wrap.getStatus();
+  if (status < kInstantiated)
+    wrap.instantiate();
+  if (status < kEvaluated)
+    await wrap.evaluate(-1, false);
+  return wrap.namespace();
+}
+
 module.exports = {
   Module,
-  initImportMetaMap,
-  wrapToModuleMap
+  callLinkerForNamespace,
 };

lib/repl.js

@@ -49,6 +49,7 @@ const {
 } = require('internal/modules/cjs/helpers');
 const internalUtil = require('internal/util');
 const { isTypedArray } = require('internal/util/types');
+const { getURLFromFilePath } = require('internal/url');
 const util = require('util');
 const utilBinding = process.binding('util');
 const { inherits } = util;
@@ -256,7 +257,17 @@ function REPLServer(prompt,
         }
         script = vm.createScript(code, {
           filename: file,
-          displayErrors: true
+          displayErrors: true,
+          async resolveDynamicImport(specifier, scriptOrModule) {
+            const loader = await require('internal/process/esm_loader')
+              .loaderPromise;
+
+            const filename = scriptOrModule.filename;
+            const referrer = scriptOrModule.url ||
+              getURLFromFilePath(filename === 'repl' ?
+                path.join(process.cwd(), filename) : filename).href;
+            return loader.import(specifier, referrer);
+          },
         });
       } catch (e) {
         debug('parse error %j', code, e);

lib/vm.js

@@ -50,6 +50,7 @@ class Script extends ContextifyScript {
       columnOffset = 0,
       cachedData,
       produceCachedData = false,
+      resolveDynamicImport,
       [kParsingContext]: parsingContext
     } = options;
 
@@ -67,6 +68,19 @@ class Script extends ContextifyScript {
                                      produceCachedData);
     }
 
+
+    let importModuleDynamically;
+    if (resolveDynamicImport !== undefined &&
+        typeof resolveDynamicImport !== 'function') {
+      throw new ERR_INVALID_ARG_TYPE(
+        'options.resolveDynamicImport', 'function', resolveDynamicImport);
+    } else if (resolveDynamicImport !== undefined) {
+      const { callLinkerForNamespace } = require('internal/vm/module');
+      importModuleDynamically = async (specifier) => {
+        return callLinkerForNamespace(resolveDynamicImport, specifier, this);
+      };
+    }
+
     // Calling `ReThrow()` on a native TryCatch does not generate a new
     // abort-on-uncaught-exception check. A dummy try/catch in JS land
     // protects against that.
@@ -81,6 +95,12 @@ class Script extends ContextifyScript {
     } catch (e) {
       throw e; /* node-do-not-add-exception-line */
     }
+
+    if (importModuleDynamically !== undefined) {
+      require('internal/process/esm_loader')
+        .importModuleDynamicallyMap
+        .set(this, importModuleDynamically);
+    }
   }
 
   runInThisContext(options) {

src/env-inl.h

@@ -445,6 +445,10 @@ inline double Environment::get_default_trigger_async_id() {
   return default_trigger_async_id;
 }
 
+inline int Environment::get_next_module_id() {
+  return module_id_counter_++;
+}
+
 inline double* Environment::heap_statistics_buffer() const {
   CHECK_NE(heap_statistics_buffer_, nullptr);
   return heap_statistics_buffer_;