module: resolve self-references

Adds the ability to `import` or `require` a package from within its
own source code. This allows tests and examples to be written using
the package name, making them easier to reuse by consumers of the
package.

Assuming the `name` field in `package.json` is set to `my-pkg`, its
test could use `require('my-pkg')` or `import 'my-pkg'` even if
there's no `node_modules/my-pkg` while testing the package itself.

An important difference between this and relative specifiers like
`require('../')` is that self-references use the public interface
of the package as defined in the `exports` field while relative
specifiers don't.

This behavior is guarded by a new experimental flag
(`--experimental-resolve-self`).

PR-URL: #29327
Reviewed-By: Guy Bedford <guybedford@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Michaël Zasso <targos@protonmail.com>

doc/api/cli.md

@@ -196,6 +196,14 @@ added: v11.8.0
 
 Enable experimental diagnostic report feature.
 
+### `--experimental-resolve-self`
+<!-- YAML
+added: REPLACEME
+-->
+
+Enable experimental support for a package using `require` or `import` to load
+itself.
+
 ### `--experimental-vm-modules`
 <!-- YAML
 added: v9.6.0
@@ -1053,6 +1061,7 @@ Node.js options that are allowed are:
 * `--experimental-policy`
 * `--experimental-repl-await`
 * `--experimental-report`
+* `--experimental-resolve-self`
 * `--experimental-vm-modules`
 * `--experimental-wasm-modules`
 * `--force-context-aware`

doc/api/esm.md

@@ -838,9 +838,6 @@ _isMain_ is **true** when resolving the Node.js application entry point.
 > 1. Let _packageSubpath_ be *undefined*.
 > 1. If _packageSpecifier_ is an empty string, then
 >    1. Throw an _Invalid Specifier_ error.
-> 1. If _packageSpecifier_ does not start with _"@"_, then
->    1. Set _packageName_ to the substring of _packageSpecifier_ until the
->       first _"/"_ separator or the end of the string.
 > 1. Otherwise,
 >    1. If _packageSpecifier_ does not contain a _"/"_ separator, then
 >       1. Throw an _Invalid Specifier_ error.
@@ -854,7 +851,7 @@ _isMain_ is **true** when resolving the Node.js application entry point.
 >    1. Set _packageSubpath_ to _"."_ concatenated with the substring of
 >       _packageSpecifier_ from the position at the length of _packageName_.
 > 1. If _packageSubpath_ contains any _"."_ or _".."_ segments or percent
->    encoded strings for _"/"_ or _"\\"_ then,
+>    encoded strings for _"/"_ or _"\\"_, then
 >    1. Throw an _Invalid Specifier_ error.
 > 1. If _packageSubpath_ is _undefined_ and _packageName_ is a Node.js builtin
 >    module, then
@@ -877,8 +874,31 @@ _isMain_ is **true** when resolving the Node.js application entry point.
 >             1. Return **PACKAGE_EXPORTS_RESOLVE**(_packageURL_,
 >                _packageSubpath_, _pjson.exports_).
 >       1. Return the URL resolution of _packageSubpath_ in _packageURL_.
+> 1. Set _selfUrl_ to the result of
+>    **SELF_REFERENCE_RESOLE**(_packageSpecifier_, _parentURL_).
+> 1. If _selfUrl_ isn't empty, return _selfUrl_.
 > 1. Throw a _Module Not Found_ error.
 
+**SELF_REFERENCE_RESOLVE**(_specifier_, _parentURL_)
+
+> 1. Let _packageURL_ be the result of **READ_PACKAGE_SCOPE**(_parentURL_).
+> 1. If _packageURL_ is **null**, then
+>    1. Return an empty result.
+> 1. Let _pjson_ be the result of **READ_PACKAGE_JSON**(_packageURL_).
+> 1. Set _name_ to _pjson.name_.
+> 1. If _name_ is empty, then return an empty result.
+> 1. If _name_ is equal to _specifier_, then
+>    1. Return the result of **PACKAGE_MAIN_RESOLVE**(_packageURL_, _pjson_).
+> 1. If _specifier_ starts with _name_ followed by "/", then
+>    1. Set _subpath_ to everything after the "/".
+>    1. If _pjson_ is not **null** and _pjson_ has an _"exports"_ key, then
+>       1. Let _exports_ be _pjson.exports_.
+>       1. If _exports_ is not **null** or **undefined**, then
+>          1. Return **PACKAGE_EXPORTS_RESOLVE**(_packageURL_, _subpath_,
+>             _pjson.exports_).
+>    1. Return the URL resolution of _subpath_ in _packageURL_.
+> 1. Otherwise return an empty result.
+
 **PACKAGE_MAIN_RESOLVE**(_packageURL_, _pjson_)
 
 > 1. If _pjson_ is **null**, then

doc/api/modules.md

@@ -160,8 +160,9 @@ require(X) from module at path Y
    a. LOAD_AS_FILE(Y + X)
    b. LOAD_AS_DIRECTORY(Y + X)
    c. THROW "not found"
-4. LOAD_NODE_MODULES(X, dirname(Y))
-5. THROW "not found"
+5. LOAD_NODE_MODULES(X, dirname(Y))
+4. LOAD_SELF_REFERENCE(X, dirname(Y))
+6. THROW "not found"
 
 LOAD_AS_FILE(X)
 1. If X is a file, load X as JavaScript text.  STOP
@@ -201,6 +202,13 @@ NODE_MODULES_PATHS(START)
    c. DIRS = DIRS + DIR
    d. let I = I - 1
 5. return DIRS
+
+LOAD_SELF_REFERENCE(X, START)
+1. Find the closest package scope to START.
+2. If no scope was found, throw "not found".
+3. If the name in `package.json` isn't a prefix of X, throw "not found".
+4. Otherwise, resolve the remainder of X relative to this package as if it
+   was loaded via `LOAD_NODE_MODULES` with a name in `package.json`.
 ```
 
 Node.js allows packages loaded via

lib/internal/modules/cjs/loader.js

@@ -59,6 +59,7 @@ const enableSourceMaps = getOptionValue('--enable-source-maps');
 const preserveSymlinks = getOptionValue('--preserve-symlinks');
 const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main');
 const experimentalModules = getOptionValue('--experimental-modules');
+const experimentalSelf = getOptionValue('--experimental-resolve-self');
 const manifest = getOptionValue('--experimental-policy') ?
   require('internal/process/policy').manifest :
   null;
@@ -237,6 +238,7 @@ function readPackage(requestPath) {
   try {
     const parsed = JSON.parse(json);
     const filtered = {
+      name: parsed.name,
       main: parsed.main,
       exports: parsed.exports,
       type: parsed.type
@@ -366,6 +368,125 @@ function findLongestRegisteredExtension(filename) {
   return '.js';
 }
 
+function resolveBasePath(basePath, exts, isMain, trailingSlash, request) {
+  let filename;
+
+  const rc = stat(basePath);
+  if (!trailingSlash) {
+    if (rc === 0) {  // File.
+      if (!isMain) {
+        if (preserveSymlinks) {
+          filename = path.resolve(basePath);
+        } else {
+          filename = toRealPath(basePath);
+        }
+      } else if (preserveSymlinksMain) {
+        // For the main module, we use the preserveSymlinksMain flag instead
+        // mainly for backward compatibility, as the preserveSymlinks flag
+        // historically has not applied to the main module.  Most likely this
+        // was intended to keep .bin/ binaries working, as following those
+        // symlinks is usually required for the imports in the corresponding
+        // files to resolve; that said, in some use cases following symlinks
+        // causes bigger problems which is why the preserveSymlinksMain option
+        // is needed.
+        filename = path.resolve(basePath);
+      } else {
+        filename = toRealPath(basePath);
+      }
+    }
+
+    if (!filename) {
+      // Try it with each of the extensions
+      if (exts === undefined)
+        exts = Object.keys(Module._extensions);
+      filename = tryExtensions(basePath, exts, isMain);
+    }
+  }
+
+  if (!filename && rc === 1) {  // Directory.
+    // try it with each of the extensions at "index"
+    if (exts === undefined)
+      exts = Object.keys(Module._extensions);
+    filename = tryPackage(basePath, exts, isMain, request);
+  }
+
+  return filename;
+}
+
+function trySelf(paths, exts, isMain, trailingSlash, request) {
+  if (!experimentalSelf) {
+    return false;
+  }
+
+  const { data: pkg, path: basePath } = readPackageScope(paths[0]);
+  if (!pkg) return false;
+  if (typeof pkg.name !== 'string') return false;
+
+  let expansion;
+  if (request === pkg.name) {
+    expansion = '';
+  } else if (StringPrototype.startsWith(request, `${pkg.name}/`)) {
+    expansion = StringPrototype.slice(request, pkg.name.length);
+  } else {
+    return false;
+  }
+
+  if (exts === undefined)
+    exts = Object.keys(Module._extensions);
+
+  if (expansion) {
+    // Use exports
+    const fromExports = applyExports(basePath, expansion);
+    if (!fromExports) return false;
+    return resolveBasePath(fromExports, exts, isMain, trailingSlash, request);
+  } else {
+    // Use main field
+    return tryPackage(basePath, exts, isMain, request);
+  }
+}
+
+function applyExports(basePath, expansion) {
+  const pkgExports = readPackageExports(basePath);
+  const mappingKey = `.${expansion}`;
+
+  if (typeof pkgExports === 'object' && pkgExports !== null) {
+    if (ObjectPrototype.hasOwnProperty(pkgExports, mappingKey)) {
+      const mapping = pkgExports[mappingKey];
+      return resolveExportsTarget(pathToFileURL(basePath + '/'), mapping, '',
+                                  basePath, mappingKey);
+    }
+
+    let dirMatch = '';
+    for (const candidateKey of Object.keys(pkgExports)) {
+      if (candidateKey[candidateKey.length - 1] !== '/') continue;
+      if (candidateKey.length > dirMatch.length &&
+          StringPrototype.startsWith(mappingKey, candidateKey)) {
+        dirMatch = candidateKey;
+      }
+    }
+
+    if (dirMatch !== '') {
+      const mapping = pkgExports[dirMatch];
+      const subpath = StringPrototype.slice(mappingKey, dirMatch.length);
+      return resolveExportsTarget(pathToFileURL(basePath + '/'), mapping,
+                                  subpath, basePath, mappingKey);
+    }
+  }
+  if (mappingKey === '.' && typeof pkgExports === 'string') {
+    return resolveExportsTarget(pathToFileURL(basePath + '/'), pkgExports,
+                                '', basePath, mappingKey);
+  }
+  if (pkgExports != null) {
+    // eslint-disable-next-line no-restricted-syntax
+    const e = new Error(`Package exports for '${basePath}' do not define ` +
+        `a '${mappingKey}' subpath`);
+    e.code = 'MODULE_NOT_FOUND';
+    throw e;
+  }
+
+  return path.resolve(basePath, mappingKey);
+}
+
 // This only applies to requests of a specific form:
 // 1. name/.*
 // 2. @scope/name/.*
@@ -380,43 +501,7 @@ function resolveExports(nmPath, request, absoluteRequest) {
     }
 
     const basePath = path.resolve(nmPath, name);
-    const pkgExports = readPackageExports(basePath);
-    const mappingKey = `.${expansion}`;
-
-    if (typeof pkgExports === 'object' && pkgExports !== null) {
-      if (ObjectPrototype.hasOwnProperty(pkgExports, mappingKey)) {
-        const mapping = pkgExports[mappingKey];
-        return resolveExportsTarget(pathToFileURL(basePath + '/'), mapping, '',
-                                    basePath, mappingKey);
-      }
-
-      let dirMatch = '';
-      for (const candidateKey of Object.keys(pkgExports)) {
-        if (candidateKey[candidateKey.length - 1] !== '/') continue;
-        if (candidateKey.length > dirMatch.length &&
-            StringPrototype.startsWith(mappingKey, candidateKey)) {
-          dirMatch = candidateKey;
-        }
-      }
-
-      if (dirMatch !== '') {
-        const mapping = pkgExports[dirMatch];
-        const subpath = StringPrototype.slice(mappingKey, dirMatch.length);
-        return resolveExportsTarget(pathToFileURL(basePath + '/'), mapping,
-                                    subpath, basePath, mappingKey);
-      }
-    }
-    if (mappingKey === '.' && typeof pkgExports === 'string') {
-      return resolveExportsTarget(pathToFileURL(basePath + '/'), pkgExports,
-                                  '', basePath, mappingKey);
-    }
-    if (pkgExports != null) {
-      // eslint-disable-next-line no-restricted-syntax
-      const e = new Error(`Package exports for '${basePath}' do not define ` +
-          `a '${mappingKey}' subpath`);
-      e.code = 'MODULE_NOT_FOUND';
-      throw e;
-    }
+    return applyExports(basePath, expansion);
   }
 
   return path.resolve(nmPath, request);
@@ -532,6 +617,13 @@ Module._findPath = function(request, paths, isMain) {
       return filename;
     }
   }
+
+  const selfFilename = trySelf(paths, exts, isMain, trailingSlash, request);
+  if (selfFilename) {
+    Module._pathCache[cacheKey] = selfFilename;
+    return selfFilename;
+  }
+
   return false;
 };
 

src/env.h

@@ -93,12 +93,15 @@ struct PackageConfig {
   enum class Exists { Yes, No };
   enum class IsValid { Yes, No };
   enum class HasMain { Yes, No };
+  enum class HasName { Yes, No };
   enum PackageType : uint32_t { None = 0, CommonJS, Module };
 
   const Exists exists;
   const IsValid is_valid;
   const HasMain has_main;
   const std::string main;
+  const HasName has_name;
+  const std::string name;
   const PackageType type;
 
   v8::Global<v8::Value> exports;