feat(module-source): Add XS specific variant (#2470)

Closes: #2250
Refs: #2251

## Description

Toward using native XS compartments and module sources, this change adds
an XS-specific variant of `@endo/module-source` that patches the native
`ModuleSource` in place instead of entraining Babel. This will allow an
XS-specific `@endo/import-bundle` variant to load both pre-compiled
sources (`pre-json-mjs`) and original ESM sources (`mjs`).

This change is opt-in, provided the `xs` runtime condition. These are
conditions introduced by Node.js with the `node -C development entry.js`
precedent and supported by `@endo/compartment-mapper` for both
`endoScript` and `endoZipBase64` bundle formats.

### Security Considerations

This change increases exposure to a native ModuleSource as implemented
in a memory-unsafe language.

### Scaling Considerations

We expect a significant improvement in performance at both bundle time
and run time upon completion of #2251 since native ModuleSource is
considerably faster than a virtualization.

### Documentation Considerations

Captured in NEWS.md and README.md.

### Testing Considerations

This change introduces a minimal `text:xs` that runs with `xst` and is
exercised by Continuous Integration. This minimal test can be expanded
as we make more progress with the XS-variant of SES, which will allow
the virtual Compartment to opt-in to use native ModuleSource.

### Compatibility Considerations

The XS-specific shim is not a one-to-one replacement for the Babel
implementation and does not support some of the optional arguments. Code
that expects these hooks to be called will be dismayed. Despite these
limitations, no existing program should break: the `xs` condition should
not be present in any existing application.

### Upgrade Considerations

This change will not come to bear on the Agoric chain until after a new
version of XS becomes available with a new XS-specific lockdown
preamble.

packages/eslint-plugin/lib/configs/internal.js

@@ -68,6 +68,7 @@ module.exports = {
     'bundles/**',
     'coverage/**',
     'dist/**',
+    'tmp/**',
     'test262/**',
     'ava*.config.js',
   ],

packages/module-source/.gitignore

@@ -0,0 +1 @@
+tmp

packages/module-source/NEWS.md

@@ -1,5 +1,10 @@
 User-visible changes in `@endo/module-source`:
 
+# Next release
+
+- Provides an XS-specific variant of `@endo/module-source` that adapts the
+  native `ModuleSource` instead of entraining Babel.
+
 # v1.1.0 (2024-10-10)
 
 - Adds `@endo/module-source/shim.js` to shim `globalThis.ModuleSource`.

packages/module-source/README.md

@@ -74,6 +74,15 @@ the hook returns a promise, it will be dropped and rejections will go uncaught.
 If the hook must do async work, these should be queued up as a job that the
 caller can later await.
 
+## XS Specific Variant
+
+With the `xs` condition, `@endo/module-source` will not entrain Babel and will
+just adapt the native `ModuleSource` to the older interface presented by this
+package.
+That is, the XS native `bindings` will be translated to `imports`, `exports`,
+and `reexports` getters.
+This form of `ModuleSource` ignores all options.
+
 ## Bug Disclosure
 
 Please help us practice coordinated security bug disclosure, by using the

packages/module-source/package.json

@@ -22,7 +22,10 @@
   "type": "module",
   "main": "./index.js",
   "exports": {
-    ".": "./index.js",
+    ".": {
+      "xs": "./src-xs/index.js",
+      "default": "./index.js"
+    },
     "./shim.js": "./shim.js",
     "./package.json": "./package.json"
   },
@@ -35,7 +38,8 @@
     "lint:types": "tsc",
     "lint:eslint": "eslint .",
     "lint-fix": "eslint --fix .",
-    "test": "ava"
+    "test": "ava",
+    "test:xs": "node scripts/generate-test-xs.js && xst tmp/test-xs.js"
   },
   "dependencies": {
     "@agoric/babel-generator": "^7.17.6",

packages/module-source/scripts/generate-test-xs.js

@@ -0,0 +1,41 @@
+/* global process */
+import 'ses';
+import { promises as fs } from 'fs';
+// Lerna does not like dependency cycles.
+// With an explicit devDependency from module-source to compartment-mapper,
+// the build script stalls before running every package's build script.
+//   yarn lerna run build
+// Omitting the dependency from package.json solves the problem and works
+// by dint of shared workspace node_modules.
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { makeBundle } from '@endo/compartment-mapper/bundle.js';
+import { fileURLToPath } from 'url';
+
+const read = async location => {
+  const path = fileURLToPath(location);
+  return fs.readFile(path);
+};
+const write = async (location, content) => {
+  const path = fileURLToPath(location);
+  await fs.writeFile(path, content);
+};
+
+const main = async () => {
+  const xsPrelude = await makeBundle(
+    read,
+    new URL('../test/_xs.js', import.meta.url).href,
+    {
+      tags: new Set(['xs']),
+    },
+  );
+
+  await fs.mkdir(fileURLToPath(new URL('../tmp', import.meta.url)), {
+    recursive: true,
+  });
+  await write(new URL('../tmp/test-xs.js', import.meta.url).href, xsPrelude);
+};
+
+main().catch(err => {
+  console.error('Error running main:', err);
+  process.exitCode = 1;
+});

packages/module-source/shim.js

@@ -1,6 +1,10 @@
 /* global globalThis */
 
-import { ModuleSource } from './index.js';
+// We are using a reflexive import to make sure we pass through the conditional
+// export in package.json.
+// Eslint does not yet seem to have a carve-out for package-reflexive imports.
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { ModuleSource } from '@endo/module-source';
 
 Object.defineProperty(globalThis, 'ModuleSource', {
   value: ModuleSource,

packages/module-source/src-xs/index.js

@@ -0,0 +1,67 @@
+// @ts-check
+/* global globalThis */
+/* eslint-disable @endo/no-nullish-coalescing */
+
+/**
+ * @typedef {|
+ *   { import: string, as?: string, from: string } |
+ *   { importAllFrom: string, as: string, from: string } |
+ *   { export: string, as?: string, from?: string } |
+ *   { exportAllFrom: string, as?: string } |
+ *   {importFrom: string }
+ * } Binding
+ */
+
+/** @param {Binding[]} bindings */
+function* getImports(bindings) {
+  for (const binding of bindings) {
+    if (binding.import !== undefined) {
+      yield binding.from;
+    } else if (binding.importFrom !== undefined) {
+      yield binding.importFrom;
+    } else if (binding.importAllFrom !== undefined) {
+      yield binding.importAllFrom;
+    } else if (binding.exportAllFrom !== undefined) {
+      yield binding.exportAllFrom;
+    }
+  }
+}
+
+/** @param {Binding[]} bindings */
+function* getExports(bindings) {
+  for (const binding of bindings) {
+    if (binding.export !== undefined) {
+      yield binding.as ?? binding.export;
+    }
+  }
+}
+
+/** @param {Binding[]} bindings */
+function* getReexports(bindings) {
+  for (const binding of bindings) {
+    if (binding.exportAllFrom !== undefined) {
+      yield binding.exportAllFrom;
+    }
+  }
+}
+
+const ModuleSource = globalThis.ModuleSource;
+
+Object.defineProperties(
+  ModuleSource.prototype,
+  Object.getOwnPropertyDescriptors({
+    get imports() {
+      return Array.from(new Set(getImports(this.bindings)));
+    },
+
+    get exports() {
+      return Array.from(new Set(getExports(this.bindings)));
+    },
+
+    get reexports() {
+      return Array.from(new Set(getReexports(this.bindings)));
+    },
+  }),
+);
+
+export { ModuleSource };

packages/module-source/test/_native.js

@@ -0,0 +1,3 @@
+/* global globalThis */
+export const NativeModuleSource = globalThis.ModuleSource;
+export const NativeCompartment = globalThis.Compartment;

packages/module-source/test/_xs.js

@@ -0,0 +1,41 @@
+// This is a test fixture for XS validation (yarn test:xs with Moddable's xst
+// on the PATH)
+// This must be bundled with the -C xs condition to produce an artifact
+// (tmp/test-xs.js) suitable for running with xst.
+import { NativeModuleSource, NativeCompartment } from './_native.js';
+// Eslint does not know about package reflexive imports (importing your own
+// package), which in this case is necessary to go through the conditional
+// export in package.json.
+// eslint-disable-next-line import/no-extraneous-dependencies
+import '@endo/module-source/shim.js';
+import 'ses';
+
+lockdown();
+
+// spot checks
+assert(Object.isFrozen(Object));
+
+const source = new ModuleSource(`
+  import name from 'imported';
+  export * from 'reexported';
+  export default 42;
+  throw new Error('unreached');
+`);
+assert(source.imports[0] === 'imported');
+assert(source.exports[0] === 'default');
+assert(source.reexports[0] === 'reexported');
+
+assert(source instanceof NativeModuleSource);
+
+const compartment = new NativeCompartment({
+  modules: {
+    '.': {
+      source: new ModuleSource(`
+        export default 42;
+      `),
+    },
+  },
+});
+assert(compartment.importNow('.').default === 42);
+
+// to be continued with XS-specific adapters for Compartment in SES...

packages/ses/src/error/console.js

@@ -288,8 +288,10 @@ export const makeCausalConsole = (baseConsole, loggedErrorHandler) => {
         logError(severity, subError);
       }
     } finally {
-      // eslint-disable-next-line @endo/no-polymorphic-call
-      baseConsole.groupEnd();
+      if (baseConsole.groupEnd) {
+        // eslint-disable-next-line @endo/no-polymorphic-call
+        baseConsole.groupEnd();
+      }
     }
   };
 

packages/ses/src/permits.js

@@ -1524,6 +1524,10 @@ export const permitted = {
     bindings: getter,
     needsImport: getter,
     needsImportMeta: getter,
+    // @endo/module-source provides a legacy interface
+    imports: getter,
+    exports: getter,
+    reexports: getter,
   },
 
   '%AbstractModuleSource%': {

packages/ses/src/reporting-types.d.ts

@@ -6,8 +6,8 @@ export type Reporter = {
 };
 
 export type GroupReporter = Reporter & {
-  groupCollapsed: (label: string) => void;
-  groupEnd: () => void;
+  groupCollapsed?: (label: string) => void;
+  groupEnd?: () => void;
 };
 
 // Console implements GroupReporter

packages/ses/src/reporting.js

@@ -80,27 +80,29 @@ export const chooseReporter = reporting => {
  */
 export const reportInGroup = (groupLabel, console, callback) => {
   const { warn, error, groupCollapsed, groupEnd } = console;
+  const grouping = groupCollapsed && groupEnd;
   let groupStarted = false;
   try {
     return callback({
       warn(...args) {
-        if (!groupStarted) {
+        if (grouping && !groupStarted) {
           groupCollapsed(groupLabel);
           groupStarted = true;
         }
         warn(...args);
       },
       error(...args) {
-        if (!groupStarted) {
+        if (grouping && !groupStarted) {
           groupCollapsed(groupLabel);
           groupStarted = true;
         }
         error(...args);
       },
     });
   } finally {
-    if (groupStarted) {
+    if (grouping && groupStarted) {
       groupEnd();
+      groupStarted = false;
     }
   }
 };