esm: --experimental-default-type flag to flip module defaults

PR-URL: nodejs/node#49869
Backport-PR-URL: nodejs/node#50669
Reviewed-By: Guy Bedford <guybedford@gmail.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

graal-nodejs/doc/api/cli.md

@@ -366,6 +366,36 @@ added: v17.6.0
 
 Expose the [Web Crypto API][] on the global scope.
 
+### `--experimental-default-type=type`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+Define which module system, `module` or `commonjs`, to use for the following:
+
+* String input provided via `--eval` or STDIN, if `--input-type` is unspecified.
+
+* Files ending in `.js` or with no extension, if there is no `package.json` file
+  present in the same folder or any parent folder.
+
+* Files ending in `.js` or with no extension, if the nearest parent
+  `package.json` field lacks a `"type"` field; unless the `package.json` folder
+  or any parent folder is inside a `node_modules` folder.
+
+In other words, `--experimental-default-type=module` flips all the places where
+Node.js currently defaults to CommonJS to instead default to ECMAScript modules,
+with the exception of folders and subfolders below `node_modules`, for backward
+compatibility.
+
+Under `--experimental-default-type=module` and `--experimental-wasm-modules`,
+files with no extension will be treated as WebAssembly if they begin with the
+WebAssembly magic number (`\0asm`); otherwise they will be treated as ES module
+JavaScript.
+
 ### `--experimental-import-meta-resolve`
 
 <!-- YAML
@@ -1923,6 +1953,7 @@ Node.js options that are allowed are:
 * `--enable-network-family-autoselection`
 * `--enable-source-maps`
 * `--experimental-abortcontroller`
+* `--experimental-default-type`
 * `--experimental-global-customevent`
 * `--experimental-global-webcrypto`
 * `--experimental-import-meta-resolve`

graal-nodejs/doc/api/esm.md

@@ -105,10 +105,11 @@ provides interoperability between them and its original module format,
 
 Node.js has two module systems: [CommonJS][] modules and ECMAScript modules.
 
-Authors can tell Node.js to use the ECMAScript modules loader
-via the `.mjs` file extension, the `package.json` [`"type"`][] field, or the
-[`--input-type`][] flag. Outside of those cases, Node.js will use the CommonJS
-module loader. See [Determining module system][] for more details.
+Authors can tell Node.js to use the ECMAScript modules loader via the `.mjs`
+file extension, the `package.json` [`"type"`][] field, the [`--input-type`][]
+flag, or the [`--experimental-default-type`][] flag. Outside of those cases,
+Node.js will use the CommonJS module loader. See [Determining module system][]
+for more details.
 
 <!-- Anchors to make sure old links find a target -->
 
@@ -1080,6 +1081,7 @@ success!
 [URL]: https://url.spec.whatwg.org/
 [`"exports"`]: packages.md#exports
 [`"type"`]: packages.md#type
+[`--experimental-default-type`]: cli.md#--experimental-default-typetype
 [`--input-type`]: cli.md#--input-typetype
 [`data:` URLs]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
 [`export`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/export

graal-nodejs/doc/api/packages.md

@@ -55,6 +55,8 @@ along with a reference for the [`package.json`][] fields defined by Node.js.
 
 ## Determining module system
 
+### Introduction
+
 Node.js will treat the following as [ES modules][] when passed to `node` as the
 initial input, or when referenced by `import` statements or `import()`
 expressions:
@@ -67,14 +69,9 @@ expressions:
 * Strings passed in as an argument to `--eval`, or piped to `node` via `STDIN`,
   with the flag `--input-type=module`.
 
-Node.js will treat as [CommonJS][] all other forms of input, such as `.js` files
-where the nearest parent `package.json` file contains no top-level `"type"`
-field, or string input without the flag `--input-type`. This behavior is to
-preserve backward compatibility. However, now that Node.js supports both
-CommonJS and ES modules, it is best to be explicit whenever possible. Node.js
-will treat the following as CommonJS when passed to `node` as the initial input,
-or when referenced by `import` statements, `import()` expressions, or
-`require()` expressions:
+Node.js will treat the following as [CommonJS][] when passed to `node` as the
+initial input, or when referenced by `import` statements or `import()`
+expressions:
 
 * Files with a `.cjs` extension.
 
@@ -84,11 +81,30 @@ or when referenced by `import` statements, `import()` expressions, or
 * Strings passed in as an argument to `--eval` or `--print`, or piped to `node`
   via `STDIN`, with the flag `--input-type=commonjs`.
 
-Package authors should include the [`"type"`][] field, even in packages where
-all sources are CommonJS. Being explicit about the `type` of the package will
-future-proof the package in case the default type of Node.js ever changes, and
-it will also make things easier for build tools and loaders to determine how the
-files in the package should be interpreted.
+Aside from these explicit cases, there are other cases where Node.js defaults to
+one module system or the other based on the value of the
+[`--experimental-default-type`][] flag:
+
+* Files ending in `.js` or with no extension, if there is no `package.json` file
+  present in the same folder or any parent folder.
+
+* Files ending in `.js` or with no extension, if the nearest parent
+  `package.json` field lacks a `"type"` field; unless the folder is inside a
+  `node_modules` folder. (Package scopes under `node_modules` are always treated
+  as CommonJS when the `package.json` file lacks a `"type"` field, regardless
+  of `--experimental-default-type`, for backward compatibility.)
+
+* Strings passed in as an argument to `--eval` or piped to `node` via `STDIN`,
+  when `--input-type` is unspecified.
+
+This flag currently defaults to `"commonjs"`, but it may change in the future to
+default to `"module"`. For this reason it is best to be explicit wherever
+possible; in particular, package authors should always include the [`"type"`][]
+field in their `package.json` files, even in packages where all sources are
+CommonJS. Being explicit about the `type` of the package will future-proof the
+package in case the default type of Node.js ever changes, and it will also make
+things easier for build tools and loaders to determine how the files in the
+package should be interpreted.
 
 ### Modules loaders
 
@@ -1337,6 +1353,7 @@ This field defines [subpath imports][] for the current package.
 [`"packageManager"`]: #packagemanager
 [`"type"`]: #type
 [`--conditions` / `-C` flag]: #resolving-user-conditions
+[`--experimental-default-type`]: cli.md#--experimental-default-typetype
 [`--no-addons` flag]: cli.md#--no-addons
 [`ERR_PACKAGE_PATH_NOT_EXPORTED`]: errors.md#err_package_path_not_exported
 [`esm`]: https://github.com/standard-things/esm#readme

graal-nodejs/doc/node.1

@@ -140,6 +140,11 @@ Requires Node.js to be built with
 .It Fl -enable-source-maps
 Enable Source Map V3 support for stack traces.
 .
+.It Fl -experimental-default-type Ns = Ns Ar type
+Interpret as either ES modules or CommonJS modules input via --eval or STDIN, when --input-type is unspecified;
+.js or extensionless files with no sibling or parent package.json;
+.js or extensionless files whose nearest parent package.json lacks a "type" field, unless under node_modules.
+.
 .It Fl -experimental-global-customevent
 Expose the CustomEvent on the global scope.
 .

graal-nodejs/lib/internal/main/check_syntax.js

@@ -63,7 +63,8 @@ function loadESMIfNeeded(cb) {
 async function checkSyntax(source, filename) {
   let isModule = true;
   if (filename === '[stdin]' || filename === '[eval]') {
-    isModule = getOptionValue('--input-type') === 'module';
+    isModule = getOptionValue('--input-type') === 'module' ||
+      (getOptionValue('--experimental-default-type') === 'module' && getOptionValue('--input-type') !== 'commonjs');
   } else {
     const { defaultResolve } = require('internal/modules/esm/resolve');
     const { defaultGetFormat } = require('internal/modules/esm/get_format');

graal-nodejs/lib/internal/main/eval_stdin.js

@@ -25,12 +25,14 @@ readStdin((code) => {
 
   const print = getOptionValue('--print');
   const loadESM = getOptionValue('--import').length > 0;
-  if (getOptionValue('--input-type') === 'module')
+  if (getOptionValue('--input-type') === 'module' ||
+    (getOptionValue('--experimental-default-type') === 'module' && getOptionValue('--input-type') !== 'commonjs')) {
     evalModule(code, print);
-  else
+  } else {
     evalScript('[stdin]',
                code,
                getOptionValue('--inspect-brk'),
                print,
                loadESM);
+  }
 });

graal-nodejs/lib/internal/main/eval_string.js

@@ -23,11 +23,13 @@ markBootstrapComplete();
 const source = getOptionValue('--eval');
 const print = getOptionValue('--print');
 const loadESM = getOptionValue('--import').length > 0 || getOptionValue('--experimental-loader').length > 0;
-if (getOptionValue('--input-type') === 'module')
+if (getOptionValue('--input-type') === 'module' ||
+  (getOptionValue('--experimental-default-type') === 'module' && getOptionValue('--input-type') !== 'commonjs')) {
   evalModule(source, print);
-else
+} else {
   evalScript('[eval]',
              source,
              getOptionValue('--inspect-brk'),
              print,
              loadESM);
+}

graal-nodejs/lib/internal/modules/esm/formats.js

@@ -2,9 +2,11 @@
 
 const {
   RegExpPrototypeExec,
+  Uint8Array,
 } = primordials;
 const { getOptionValue } = require('internal/options');
 
+const { closeSync, openSync, readSync } = require('fs');
 
 const experimentalWasmModules = getOptionValue('--experimental-wasm-modules');
 
@@ -49,8 +51,34 @@ function getLegacyExtensionFormat(ext) {
   return legacyExtensionFormatMap[ext];
 }
 
+/**
+ * For extensionless files in a `module` package scope, or a default `module` scope enabled by the
+ * `--experimental-default-type` flag, we check the file contents to disambiguate between ES module JavaScript and Wasm.
+ * We do this by taking advantage of the fact that all Wasm files start with the header `0x00 0x61 0x73 0x6d` (`_asm`).
+ * @param {URL} url
+ */
+function getFormatOfExtensionlessFile(url) {
+  if (!experimentalWasmModules) { return 'module'; }
+
+  const magic = new Uint8Array(4);
+  let fd;
+  try {
+    // TODO(@anonrig): Optimize the following by having a single C++ call
+    fd = openSync(url);
+    readSync(fd, magic, 0, 4); // Only read the first four bytes
+    if (magic[0] === 0x00 && magic[1] === 0x61 && magic[2] === 0x73 && magic[3] === 0x6d) {
+      return 'wasm';
+    }
+  } finally {
+    if (fd !== undefined) { closeSync(fd); }
+  }
+
+  return 'module';
+}
+
 module.exports = {
   extensionFormatMap,
+  getFormatOfExtensionlessFile,
   getLegacyExtensionFormat,
   legacyExtensionFormatMap,
   mimeToFormat,

graal-nodejs/lib/internal/modules/esm/get_format.js

@@ -1,16 +1,19 @@
 'use strict';
+
 const {
   RegExpPrototypeExec,
   ObjectPrototypeHasOwnProperty,
   PromisePrototypeThen,
   PromiseResolve,
+  StringPrototypeIncludes,
   StringPrototypeCharCodeAt,
   StringPrototypeSlice,
 } = primordials;
 const { basename, relative } = require('path');
 const { getOptionValue } = require('internal/options');
 const {
   extensionFormatMap,
+  getFormatOfExtensionlessFile,
   getLegacyExtensionFormat,
   mimeToFormat,
 } = require('internal/modules/esm/formats');
@@ -19,6 +22,9 @@ const experimentalNetworkImports =
   getOptionValue('--experimental-network-imports');
 const experimentalSpecifierResolution =
   getOptionValue('--experimental-specifier-resolution');
+const defaultTypeFlag = getOptionValue('--experimental-default-type');
+// The next line is where we flip the default to ES modules someday.
+const defaultType = defaultTypeFlag === 'module' ? 'module' : 'commonjs';
 const { getPackageType, getPackageScopeConfig } = require('internal/modules/esm/resolve');
 const { fileURLToPath } = require('internal/url');
 const { ERR_UNKNOWN_FILE_EXTENSION } = require('internal/errors').codes;
@@ -69,6 +75,18 @@ function extname(url) {
   return '';
 }
 
+/**
+ * Determine whether the given file URL is under a `node_modules` folder.
+ * This function assumes that the input has already been verified to be a `file:` URL,
+ * and is a file rather than a folder.
+ * @param {URL} url
+ */
+function underNodeModules(url) {
+  if (url.protocol !== 'file:') { return false; } // We determine module types for other protocols based on MIME header
+
+  return StringPrototypeIncludes(url.pathname, '/node_modules/');
+}
+
 /**
  * @param {URL} url
  * @param {{parentURL: string}} context
@@ -77,8 +95,37 @@ function extname(url) {
  */
 function getFileProtocolModuleFormat(url, context, ignoreErrors) {
   const ext = extname(url);
+
   if (ext === '.js') {
-    return getPackageType(url) === 'module' ? 'module' : 'commonjs';
+    const packageType = getPackageType(url);
+    if (packageType !== 'none') {
+      return packageType;
+    }
+    // The controlling `package.json` file has no `type` field.
+    if (defaultType === 'module') {
+      // An exception to the type flag making ESM the default everywhere is that package scopes under `node_modules`
+      // should retain the assumption that a lack of a `type` field means CommonJS.
+      return underNodeModules(url) ? 'commonjs' : 'module';
+    }
+    return 'commonjs';
+  }
+
+  if (ext === '') {
+    const packageType = getPackageType(url);
+    if (defaultType === 'commonjs') { // Legacy behavior
+      if (packageType === 'none' || packageType === 'commonjs') {
+        return 'commonjs';
+      }
+      // If package type is `module`, fall through to the error case below
+    } else { // Else defaultType === 'module'
+      if (underNodeModules(url)) { // Exception for package scopes under `node_modules`
+        return 'commonjs';
+      }
+      if (packageType === 'none' || packageType === 'module') {
+        return getFormatOfExtensionlessFile(url);
+      } // Else packageType === 'commonjs'
+      return 'commonjs';
+    }
   }
 
   const format = extensionFormatMap[ext];
@@ -93,12 +140,10 @@ function getFileProtocolModuleFormat(url, context, ignoreErrors) {
       const config = getPackageScopeConfig(url);
       const fileBasename = basename(filepath);
       const relativePath = StringPrototypeSlice(relative(config.pjsonPath, filepath), 1);
-      suggestion = 'Loading extensionless files is not supported inside of ' +
-        '"type":"module" package.json contexts. The package.json file ' +
-        `${config.pjsonPath} caused this "type":"module" context. Try ` +
-        `changing ${filepath} to have a file extension. Note the "bin" ` +
-        'field of package.json can point to a file with an extension, for example ' +
-        `{"type":"module","bin":{"${fileBasename}":"${relativePath}.js"}}`;
+      suggestion = 'Loading extensionless files is not supported inside of "type":"module" package.json contexts ' +
+        `without --experimental-default-type=module. The package.json file ${config.pjsonPath} caused this "type":"module" ` +
+        `context. Try changing ${filepath} to have a file extension. Note the "bin" field of package.json can point ` +
+        `to a file with an extension, for example {"type":"module","bin":{"${fileBasename}":"${relativePath}.js"}}`;
     }
     throw new ERR_UNKNOWN_FILE_EXTENSION(ext, filepath, suggestion);
   }

graal-nodejs/lib/internal/modules/esm/resolve.js

@@ -36,7 +36,7 @@ const preserveSymlinks = getOptionValue('--preserve-symlinks');
 const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main');
 const experimentalNetworkImports =
   getOptionValue('--experimental-network-imports');
-const typeFlag = getOptionValue('--input-type');
+const inputTypeFlag = getOptionValue('--input-type');
 const { URL, pathToFileURL, fileURLToPath, isURL, toPathIfFileURL } = require('internal/url');
 const { canParse: URLCanParse } = internalBinding('url');
 const {
@@ -1183,7 +1183,7 @@ function defaultResolve(specifier, context = {}) {
     // input, to avoid user confusion over how expansive the effect of the
     // flag should be (i.e. entry point only, package scope surrounding the
     // entry point, etc.).
-    if (typeFlag) { throw new ERR_INPUT_TYPE_NOT_ALLOWED(); }
+    if (inputTypeFlag) { throw new ERR_INPUT_TYPE_NOT_ALLOWED(); }
   }
 
   conditions = getConditionsSet(conditions);

graal-nodejs/lib/internal/modules/run_main.js

@@ -52,13 +52,14 @@ function shouldUseESMLoader(mainPath) {
   if (esModuleSpecifierResolution === 'node') {
     return true;
   }
-  // Determine the module format of the main
+  // Determine the module format of the entry point.
   if (mainPath && StringPrototypeEndsWith(mainPath, '.mjs')) { return true; }
   if (!mainPath || StringPrototypeEndsWith(mainPath, '.cjs')) { return false; }
 
   const { readPackageScope } = require('internal/modules/package_json_reader');
   const pkg = readPackageScope(mainPath);
-  return pkg && pkg.data.type === 'module';
+  // No need to guard `pkg` as it can only be an object or `false`.
+  return pkg.data?.type === 'module' || getOptionValue('--experimental-default-type') === 'module';
 }
 
 /**