module: refactor and clarify async loader hook customizations

- This updates the comments that assume loader hooks must be async
- Differentiate the sync/async loader hook paths in naming
  `#customizations` is now `#asyncLoaderHooks` to make it clear
  it's from the async APIs.
- Differentiate the paths running on the loader hook thread
  (affects the loading of async other loader hooks and are async)
  v.s. paths on the main thread calling out to code on the loader
  hook thread (do not handle loading of other async loader hooks, and
  can be sync by blocking).
  - `Hooks` is now `AsyncLoaderHooksOnLoaderHookWorker`
  - `CustomizedModuleLoader` is now
    `AsyncLoaderHooksProxiedToLoaderHookWorker` and moved into
    `lib/internal/modules/esm/hooks.js` as it implements the same
    interface as `AsyncLoaderHooksOnLoaderHookWorker`
  - `HooksProxy` is now `AsyncLoaderHookWorker`
  - Adjust the JSDoc accordingly
- Clarify the "loader worker" as the "async loader hook worker"
  i.e. when there's no _async_ loader hook registered, there won't
  be this worker, to avoid the misconception that this worker
  is spawned unconditionally.
- The code run on the loader hook worker to process
  `--experimental-loader` is moved into
  `lib/internal/modules/esm/worker.js` for clarity.
- The initialization configuration `forceDefaultLoader` is split
  into `shouldSpawnLoaderHookWorker` and `shouldPreloadModules`
  as those can be separate.
- `--experimental-vm-modules` is now processed during pre-execution
  and no longer part of the initialization of the built-in ESM
  loader, as it only exposes the vm APIs of ESM, and is unrelated
  to built-in ESM loading.

PR-URL: #60278
Reviewed-By: Geoffrey Booth <webadmin@geoffreybooth.com>

Author: joyeecheung

lib/internal/main/worker_thread.js

@@ -11,15 +11,14 @@ const {
   ObjectDefineProperty,
   PromisePrototypeThen,
   RegExpPrototypeExec,
-  SafeWeakMap,
   globalThis: {
     SharedArrayBuffer,
   },
 } = primordials;
 
 const {
   prepareWorkerThreadExecution,
-  setupUserModules,
+  initializeModuleLoaders,
   markBootstrapComplete,
 } = require('internal/process/pre_execution');
 
@@ -138,11 +137,13 @@ port.on('message', (message) => {
       workerIo.sharedCwdCounter = cwdCounter;
     }
 
-    const isLoaderWorker =
-      doEval === 'internal' &&
-      filename === require('internal/modules/esm/utils').loaderWorkerId;
-    // Disable custom loaders in loader worker.
-    setupUserModules(isLoaderWorker);
+    const isLoaderHookWorker = (filename === 'internal/modules/esm/worker' && doEval === 'internal');
+    if (!isLoaderHookWorker) {
+      // If we are in the loader hook worker, delay the module loader initializations until
+      // initializeAsyncLoaderHooksOnLoaderHookWorker() which needs to run preloads
+      // after the asynchronous loader hooks are registered.
+      initializeModuleLoaders({ shouldSpawnLoaderHookWorker: true, shouldPreloadModules: true });
+    }
 
     if (!hasStdin)
       process.stdin.push(null);
@@ -152,9 +153,10 @@ port.on('message', (message) => {
     port.postMessage({ type: UP_AND_RUNNING });
     switch (doEval) {
       case 'internal': {
-        // Create this WeakMap in js-land because V8 has no C++ API for WeakMap.
-        internalBinding('module_wrap').callbackMap = new SafeWeakMap();
-        require(filename)(workerData, publicPort);
+        // Currently the only user of internal eval is the async loader hook thread.
+        assert(isLoaderHookWorker, `Unexpected internal eval ${filename}`);
+        const setupModuleWorker = require('internal/modules/esm/worker');
+        setupModuleWorker(workerData, publicPort);
         break;
       }
 

lib/internal/modules/esm/hooks.js

@@ -54,7 +54,6 @@ const {
 } = require('internal/modules/esm/resolve');
 const {
   getDefaultConditions,
-  loaderWorkerId,
 } = require('internal/modules/esm/utils');
 const { deserializeError } = require('internal/error_serdes');
 const {
@@ -105,7 +104,39 @@ function defineImportAssertionAlias(context) {
 
 // [2] `validate...()`s throw the wrong error
 
-class Hooks {
+/**
+ * @typedef {{ format: ModuleFormat, source: ModuleSource }} LoadResult
+ */
+
+/**
+ * @typedef {{ format: ModuleFormat, url: string, importAttributes: Record<string, string> }} ResolveResult
+ */
+
+/**
+ * Interface for classes that implement asynchronous loader hooks that can be attached to the ModuleLoader
+ * via `ModuleLoader.#setAsyncLoaderHooks()`.
+ * @typedef {object} AsyncLoaderHooks
+ * @property {boolean} allowImportMetaResolve Whether to allow the use of `import.meta.resolve`.
+ * @property {(url: string, context: object, defaultLoad: Function) => Promise<LoadResult>} load
+ *   Calling the asynchronous `load` hook asynchronously.
+ * @property {(url: string, context: object, defaultLoad: Function) => LoadResult} loadSync
+ *   Calling the asynchronous `load` hook synchronously.
+ * @property {(originalSpecifier: string, parentURL: string,
+ *             importAttributes: Record<string, string>) => Promise<ResolveResult>} resolve
+ *   Calling the asynchronous `resolve` hook asynchronously.
+ * @property {(originalSpecifier: string, parentURL: string,
+ *            importAttributes: Record<string, string>) => ResolveResult} resolveSync
+ *   Calling the asynchronous `resolve` hook synchronously.
+ * @property {(specifier: string, parentURL: string) => any} register Register asynchronous loader hooks
+ * @property {() => void} waitForLoaderHookInitialization Force loading of hooks.
+ */
+
+/**
+ * @implements {AsyncLoaderHooks}
+ * Instances of this class run directly on the loader hook worker thread and customize the module
+ * loading of the hooks worker itself.
+ */
+class AsyncLoaderHooksOnLoaderHookWorker {
   #chains = {
     /**
      * Phase 1 of 2 in ESM loading.
@@ -452,7 +483,7 @@ class Hooks {
     };
   }
 
-  forceLoadHooks() {
+  waitForLoaderHookInitialization() {
     // No-op
   }
 
@@ -462,14 +493,20 @@ class Hooks {
     return meta;
   }
 }
-ObjectSetPrototypeOf(Hooks.prototype, null);
+ObjectSetPrototypeOf(AsyncLoaderHooksOnLoaderHookWorker.prototype, null);
 
 /**
- * There may be multiple instances of Hooks/HooksProxy, but there is only 1 Internal worker, so
- * there is only 1 MessageChannel.
+ * There is only one loader hook thread for each non-loader-hook worker thread
+ * (i.e. the non-loader-hook thread and any worker threads that are not loader hook workers themselves),
+ * so there is only 1 MessageChannel.
  */
 let MessageChannel;
-class HooksProxy {
+
+/**
+ * Abstraction over a worker thread that runs the asynchronous module loader hooks.
+ * Instances of this class run on the non-loader-hook thread and communicate with the loader hooks worker thread.
+ */
+class AsyncLoaderHookWorker {
   /**
    * Shared memory. Always use Atomics method to read or write to it.
    * @type {Int32Array}
@@ -503,7 +540,7 @@ class HooksProxy {
     const lock = new SharedArrayBuffer(SHARED_MEMORY_BYTE_LENGTH);
     this.#lock = new Int32Array(lock);
 
-    this.#worker = new InternalWorker(loaderWorkerId, {
+    this.#worker = new InternalWorker('internal/modules/esm/worker', {
       stderr: false,
       stdin: false,
       stdout: false,
@@ -644,7 +681,7 @@ class HooksProxy {
     this.#importMetaInitializer(meta, context, loader);
   }
 }
-ObjectSetPrototypeOf(HooksProxy.prototype, null);
+ObjectSetPrototypeOf(AsyncLoaderHookWorker.prototype, null);
 
 // TODO(JakobJingleheimer): Remove this when loaders go "stable".
 let globalPreloadWarningWasEmitted = false;
@@ -757,6 +794,95 @@ function nextHookFactory(current, meta, { validateArgs, validateOutput }) {
   );
 }
 
+/**
+ * @type {AsyncLoaderHookWorker}
+ * Worker instance used to run async loader hooks in a separate thread. This is a singleton for each
+ * non-loader-hook worker thread (i.e. the main thread and any worker threads that are not
+ * loader hook workers themselves).
+ */
+let asyncLoaderHookWorker;
+/**
+ * Get the AsyncLoaderHookWorker instance. If it is not defined, then create a new one.
+ * @returns {AsyncLoaderHookWorker}
+ */
+function getAsyncLoaderHookWorker() {
+  asyncLoaderHookWorker ??= new AsyncLoaderHookWorker();
+  return asyncLoaderHookWorker;
+}
+
+/**
+ * @implements {AsyncLoaderHooks}
+ * Instances of this class are created in the non-loader-hook thread and communicate with the worker thread
+ * spawned to run the async loader hooks.
+ */
+class AsyncLoaderHooksProxiedToLoaderHookWorker {
+
+  allowImportMetaResolve = true;
+
+  /**
+   * Instantiate a module loader that uses user-provided custom loader hooks.
+   */
+  constructor() {
+    getAsyncLoaderHookWorker();
+  }
+
+  /**
+   * Register some loader specifier.
+   * @param {string} originalSpecifier The specified URL path of the loader to
+   *   be registered.
+   * @param {string} parentURL The parent URL from where the loader will be
+   *   registered if using it package name as specifier
+   * @param {any} [data] Arbitrary data to be passed from the custom loader
+   *   (user-land) to the worker.
+   * @param {any[]} [transferList] Objects in `data` that are changing ownership
+   * @param {boolean} [isInternal] For internal loaders that should not be publicly exposed.
+   * @returns {{ format: string, url: URL['href'] }}
+   */
+  register(originalSpecifier, parentURL, data, transferList, isInternal) {
+    return asyncLoaderHookWorker.makeSyncRequest('register', transferList, originalSpecifier, parentURL,
+                                                 data, isInternal);
+  }
+
+  /**
+   * Resolve the location of the module.
+   * @param {string} originalSpecifier The specified URL path of the module to
+   *   be resolved.
+   * @param {string} [parentURL] The URL path of the module's parent.
+   * @param {ImportAttributes} importAttributes Attributes from the import
+   *   statement or expression.
+   * @returns {{ format: string, url: URL['href'] }}
+   */
+  resolve(originalSpecifier, parentURL, importAttributes) {
+    return asyncLoaderHookWorker.makeAsyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes);
+  }
+
+  resolveSync(originalSpecifier, parentURL, importAttributes) {
+    // This happens only as a result of `import.meta.resolve` calls, which must be sync per spec.
+    return asyncLoaderHookWorker.makeSyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes);
+  }
+
+  /**
+   * Provide source that is understood by one of Node's translators.
+   * @param {URL['href']} url The URL/path of the module to be loaded
+   * @param {object} [context] Metadata about the module
+   * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }>}
+   */
+  load(url, context) {
+    return asyncLoaderHookWorker.makeAsyncRequest('load', undefined, url, context);
+  }
+  loadSync(url, context) {
+    return asyncLoaderHookWorker.makeSyncRequest('load', undefined, url, context);
+  }
+
+  importMetaInitialize(meta, context, loader) {
+    asyncLoaderHookWorker.importMetaInitialize(meta, context, loader);
+  }
+
+  waitForLoaderHookInitialization() {
+    asyncLoaderHookWorker.waitForWorker();
+  }
+}
 
-exports.Hooks = Hooks;
-exports.HooksProxy = HooksProxy;
+exports.AsyncLoaderHooksProxiedToLoaderHookWorker = AsyncLoaderHooksProxiedToLoaderHookWorker;
+exports.AsyncLoaderHooksOnLoaderHookWorker = AsyncLoaderHooksOnLoaderHookWorker;
+exports.AsyncLoaderHookWorker = AsyncLoaderHookWorker;