feat(ses): Add XS variant of shim

packages/compartment-mapper/NEWS.md

@@ -15,6 +15,34 @@ User-visible changes to `@endo/compartment-mapper`:
   Correct interpretation of `peerDependencies` is not distributed evenly, so
   this behavior is no longer the default.
 
+Experimental:
+
+- The module `@endo/compartment-mapper/import-archive-parsers.js` does not
+  support modules in archives in their original ESM (`mjs`) or CommonJS (`cjs`)
+  formats because they entrain Babel and a full JavaScript lexer that are
+  not suitable for use in all environments, specifically XS.
+  This version introduces an elective
+  `@endo/compartment-mapper/import-archive-all-parsers.js` that has all of the
+  precompiled module parsers (`pre-cjs-json` and `pre-mjs-json`) that Endo's
+  bundler currently produces by default and additionally parsers for original
+  sources (`mjs`, `cjs`).
+  Also, provided the `xs` package condition,
+  `@endo/compartment-mapper/import-archive-parsers.js` now falls through to the
+  native `ModuleSource` and safely includes `mjs` and `cjs` without entraining
+  Babel, but is only supported in conjunction with the `__native__` option
+  for `Compartment`, `importArchive`, `parseArchive`, and `importBundle`.
+  With the `node` package condition (present by default when running ESM on
+  `node`), `@endo/compartment-mapper/import-archive-parsers.js` also now
+  includes `mjs` and `cjs` by entraining Babel, which performs adequately on
+  that platform.
+- Adds a `__native__: true` option to all paths to import, that indicates that
+  the application will fall through to the native implementation of
+  Compartment, currently only available on XS, which lacks support for
+  precompiled module sources (as exist in many archived applications,
+  particularly Agoric smart contract bundles) and instead supports loading
+  modules from original sources (which is not possible at runtime on XS).
+
+
 # v1.4.0 (2024-11-13)
 
 - Adds options `languageForExtension`, `moduleLanguageForExtension`,

packages/compartment-mapper/README.md

@@ -328,6 +328,20 @@ These will be appended to each module from the archive, for debugging purposes.
 The `@endo/bundle-source` and `@endo/import-bundle` tools integrate source maps
 for an end-to-end debugging experience.
 
+# XS (experimental)
+
+The Compartment Mapper can use native XS `Compartment` and `ModuleSource` under
+certain conditions:
+
+1. The application must be an XS script that was compiled with the `xs`
+  package condition.
+  This causes `ses`, `@endo/module-source`, and `@endo/import-bundle` to
+  provide slightly different implementations that can fall through to native
+  behavior.
+2. The application must opt-in with the `__native__: true` option on any
+  of the compartment mapper methods that import modules like `importLocation`
+  and `importArchive`.
+
 # Design
 
 Each of the workflows the compartment mapper executes a portion of one sequence

packages/compartment-mapper/import-archive-all-parsers.js

@@ -0,0 +1 @@
+export { defaultParserForLanguage } from './src/import-archive-all-parsers.js';

packages/compartment-mapper/package.json

@@ -32,7 +32,12 @@
     "./capture-lite.js": "./capture-lite.js",
     "./import-archive.js": "./import-archive.js",
     "./import-archive-lite.js": "./import-archive-lite.js",
-    "./import-archive-parsers.js": "./import-archive-parsers.js",
+    "./import-archive-parsers.js": {
+      "xs": "./import-archive-all-parsers.js",
+      "node": "./import-archive-all-parsers.js",
+      "default": "./import-archive-parsers.js"
+    },
+    "./import-archive-all-parsers.js": "./import-archive-all-parsers.js",
     "./bundle.js": "./bundle.js",
     "./node-powers.js": "./node-powers.js",
     "./node-modules.js": "./node-modules.js",

packages/compartment-mapper/src/bundle.js

@@ -117,8 +117,14 @@ const sortedModules = (
     const source = compartmentSources[compartmentName][moduleSpecifier];
     if (source !== undefined) {
       const { record, parser, deferredError, bytes } = source;
-      assert(parser !== undefined);
-      assert(bytes !== undefined);
+      assert(
+        bytes !== undefined,
+        `No bytes for ${moduleSpecifier} in ${compartmentName}`,
+      );
+      assert(
+        parser !== undefined,
+        `No parser for ${moduleSpecifier} in ${compartmentName}`,
+      );
       if (deferredError) {
         throw Error(
           `Cannot bundle: encountered deferredError ${deferredError}`,

packages/compartment-mapper/src/import-archive-all-parsers.js

@@ -0,0 +1,29 @@
+/* Provides a set of default language behaviors (parsers) suitable for
+ * evaluating archives (zip files with a `compartment-map.json` and a file for
+ * each module) with pre-compiled _or_ original ESM and CommonJS.
+ *
+ * This module does not entrain a dependency on Babel on XS, but does on other
+ * platforms like Node.js.
+ */
+/** @import {ParserForLanguage} from './types.js' */
+
+import parserPreCjs from './parse-pre-cjs.js';
+import parserJson from './parse-json.js';
+import parserText from './parse-text.js';
+import parserBytes from './parse-bytes.js';
+import parserPreMjs from './parse-pre-mjs.js';
+import parserMjs from './parse-mjs.js';
+import parserCjs from './parse-cjs.js';
+
+/** @satisfies {Readonly<ParserForLanguage>} */
+export const defaultParserForLanguage = Object.freeze(
+  /** @type {const} */ ({
+    'pre-cjs-json': parserPreCjs,
+    'pre-mjs-json': parserPreMjs,
+    cjs: parserCjs,
+    mjs: parserMjs,
+    json: parserJson,
+    text: parserText,
+    bytes: parserBytes,
+  }),
+);

packages/compartment-mapper/src/import-archive-lite.js

@@ -257,6 +257,7 @@ export const parseArchive = async (
     modules = undefined,
     importHook: exitModuleImportHook = undefined,
     parserForLanguage: parserForLanguageOption = {},
+    __native__ = false,
   } = options;
 
   const parserForLanguage = freeze(
@@ -343,6 +344,7 @@ export const parseArchive = async (
         }),
       ),
       Compartment: CompartmentParseOption,
+      __native__,
     });
 
     await pendingJobsPromise;
@@ -362,6 +364,7 @@ export const parseArchive = async (
       transforms,
       __shimTransforms__,
       Compartment: CompartmentOption = CompartmentParseOption,
+      __native__,
       importHook: exitModuleImportHook,
     } = options || {};
 
@@ -388,6 +391,7 @@ export const parseArchive = async (
       transforms,
       __shimTransforms__,
       Compartment: CompartmentOption,
+      __native__,
     });
 
     await pendingJobsPromise;

packages/compartment-mapper/src/import-lite.js

@@ -152,6 +152,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => {
       transforms,
       __shimTransforms__,
       Compartment: CompartmentOption = LoadCompartmentOption,
+      __native__,
       importHook: exitModuleImportHook,
     } = options;
     const compartmentExitModuleImportHook = exitModuleImportHookMaker({
@@ -201,6 +202,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => {
         syncModuleTransforms,
         __shimTransforms__,
         Compartment: CompartmentOption,
+        __native__,
       }));
     } else {
       // sync module transforms are allowed, because they are "compatible"
@@ -215,6 +217,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => {
         syncModuleTransforms,
         __shimTransforms__,
         Compartment: CompartmentOption,
+        __native__,
       }));
     }
 

packages/compartment-mapper/src/link.js

@@ -258,6 +258,7 @@ export const link = (
     moduleTransforms,
     syncModuleTransforms,
     __shimTransforms__ = [],
+    __native__ = false,
     archiveOnly = false,
     Compartment = defaultCompartment,
   } = options;
@@ -362,6 +363,7 @@ export const link = (
       transforms,
       __shimTransforms__,
       __options__: true,
+      __native__,
     });
 
     if (!archiveOnly) {

packages/compartment-mapper/src/types/external.ts

@@ -29,6 +29,7 @@ export type ExecuteOptions = Partial<{
   __shimTransforms__: Array<Transform>;
   attenuations: Record<string, object>;
   Compartment: typeof Compartment;
+  __native__: boolean;
 }> &
   ModulesOption &
   ExitModuleImportHookOption;
@@ -38,6 +39,7 @@ export type ParseArchiveOptions = Partial<{
   computeSha512: HashFn;
   computeSourceLocation: ComputeSourceLocationHook;
   computeSourceMapLocation: ComputeSourceMapLocationHook;
+  __native__: boolean;
 }> &
   ModulesOption &
   CompartmentOption &

packages/compartment-mapper/src/types/internal.ts

@@ -44,6 +44,7 @@ export type LinkOptions = {
   moduleTransforms?: ModuleTransforms;
   syncModuleTransforms?: SyncModuleTransforms;
   archiveOnly?: boolean;
+  __native__?: boolean;
 } & ExecuteOptions;
 
 export type LinkResult = {

packages/eslint-plugin/lib/configs/internal.js

@@ -24,7 +24,7 @@ const rules = {
 dynamicConfig.overrides.push({
   extends: ['plugin:@endo/recommended-requiring-type-checking'],
   files: fileGlobs,
-  excludedFiles: ['**/src/**/exports.js'],
+  excludedFiles: ['**/src*/**/exports.js'],
   parserOptions,
   rules,
 });

packages/ses/.gitignore

@@ -0,0 +1 @@
+tmp

packages/ses/NEWS.md

@@ -1,13 +1,23 @@
 User-visible changes in `ses`:
 
-# Next version
+# Next release
 
 - Adds support for dynamic `import` in conjunction with an update to
   `@endo/module-source`.
 
 - Specifying the long-discontinued `mathTaming` or `dateTaming` options logs a
   warning.
 
+Incubating: Please do not rely on these features as they are under development
+and subject to breaking changes that will not be signaled by semver.
+
+- Adds support for an XS-specific variant of the SES shim that is triggered
+  with the `xs` package export condition.
+  This version of SES preserves all the features of `Compartment` provided
+  uniquely by the SES shim, but with the `__native__` constructor option,
+  loses support for importing precompiled module records and gains support
+  for native `ModuleSource`.
+
 # v1.10.0 (2024-11-13)
 
 - Permit [Promise.try](https://github.com/tc39/proposal-promise-try),

packages/ses/package.json

@@ -38,6 +38,7 @@
     ".": {
       "import": {
         "types": "./types.d.ts",
+        "xs": "./src-xs/index.js",
         "default": "./index.js"
       },
       "require": {
@@ -57,8 +58,14 @@
     },
     "./tools.js": "./tools.js",
     "./assert-shim.js": "./assert-shim.js",
-    "./lockdown-shim.js": "./lockdown-shim.js",
-    "./compartment-shim.js": "./compartment-shim.js",
+    "./lockdown-shim.js": {
+      "xs": "./src-xs/lockdown-shim.js",
+      "default": "./lockdown-shim.js"
+    },
+    "./compartment-shim.js": {
+      "xs": "./src-xs/compartment-shim.js",
+      "default": "./compartment-shim.js"
+    },
     "./console-shim.js": "./console-shim.js",
     "./package.json": "./package.json"
   },
@@ -74,7 +81,7 @@
     "prepare": "npm run clean && npm run build",
     "qt": "ava",
     "test": "tsd && ava",
-    "test:xs": "xst dist/ses.umd.js test/_lockdown-safe.js",
+    "test:xs": "xst dist/ses.umd.js test/_lockdown-safe.js && node scripts/generate-test-xs.js && xst tmp/test-xs.js && rm -rf tmp",
     "postpack": "git clean -f '*.d.ts*' '*.tsbuildinfo'"
   },
   "dependencies": {

packages/ses/scripts/generate-test-xs.js

@@ -0,0 +1,56 @@
+/* eslint-env node */
+/* glimport/no-extraneous-dependenciesobal process */
+import '../index.js';
+import { promises as fs } from 'fs';
+// Lerna does not like dependency cycles.
+// With an explicit devDependency from module-source to compartment-mapper,
+// the build script stalls before running every package's build script.
+//   yarn lerna run build
+// Omitting the dependency from package.json solves the problem and works
+// by dint of shared workspace node_modules.
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { makeBundle } from '@endo/compartment-mapper/bundle.js';
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { ModuleSource } from '@endo/module-source';
+import { fileURLToPath } from 'url';
+
+const read = async location => {
+  const path = fileURLToPath(location);
+  return fs.readFile(path);
+};
+const write = async (location, content) => {
+  const path = fileURLToPath(location);
+  await fs.writeFile(path, content);
+};
+
+const main = async () => {
+  await fs.mkdir(fileURLToPath(new URL('../tmp', import.meta.url)), {
+    recursive: true,
+  });
+
+  const meaningText = await fs.readFile(
+    fileURLToPath(new URL('../test/_meaning.js', import.meta.url)),
+    'utf8',
+  );
+  const meaningModuleSource = new ModuleSource(meaningText);
+
+  await fs.writeFile(
+    fileURLToPath(new URL('../tmp/_meaning.pre-mjs.json', import.meta.url)),
+    JSON.stringify(meaningModuleSource),
+  );
+
+  const xsPrelude = await makeBundle(
+    read,
+    new URL('../test/_xs.js', import.meta.url).href,
+    {
+      tags: new Set(['xs']),
+    },
+  );
+
+  await write(new URL('../tmp/test-xs.js', import.meta.url).href, xsPrelude);
+};
+
+main().catch(err => {
+  console.error('Error running main:', err);
+  process.exitCode = 1;
+});

packages/ses/src-xs/commons.js

@@ -0,0 +1,29 @@
+/**
+ * @module In the spirit of ../src/commons.js, this module captures native
+ * functions specific to the XS engine during initialization, so vetted shims
+ * are free to modify any intrinsic without risking the integrity of SES.
+ */
+
+/// <reference types="ses"/>
+
+import {
+  getOwnPropertyDescriptor,
+  globalThis,
+  uncurryThis,
+} from '../src/commons.js';
+
+/** @type {typeof Compartment} */
+export const NativeStartCompartment = /** @type {any} */ (globalThis)
+  .Compartment;
+export const nativeCompartmentPrototype = NativeStartCompartment.prototype;
+export const nativeImport = uncurryThis(nativeCompartmentPrototype.import);
+export const nativeImportNow = uncurryThis(
+  nativeCompartmentPrototype.importNow,
+);
+/** @type {(compartment: any, source: string) => unknown} */
+export const nativeEvaluate = uncurryThis(nativeCompartmentPrototype.evaluate);
+/** @type {(compartment: typeof Compartment) => typeof globalThis} */
+export const nativeGetGlobalThis = uncurryThis(
+  // @ts-expect-error we know it is there on XS
+  getOwnPropertyDescriptor(nativeCompartmentPrototype, 'globalThis').get,
+);