feat: Elide comments in bundles

packages/bundle-source/NEWS.md

@@ -1,5 +1,12 @@
 User-visible changes to `@endo/bundle-source`:
 
+# Next version
+
+- Adds support for `--elide-comments` (`-e`) that blanks out the interior of
+  comments in `endoScript` and `endoZipBase64` formats to reduce bundle size
+  without compromising cursor position correspondence between source and bundle
+  code.
+
 # v3.3.0 (2024-07-30)
 
 - Adds support for `--no-transforms` (`-T`) which generates bundles with

packages/bundle-source/README.md

@@ -42,6 +42,21 @@ entry instead of `main` in `package.json`, if not overridden by explicit
 The `development` condition additionally implies that the bundle may import
 `devDependencies` from the package containing the entry module.
 
+## Comment Elision
+
+The `--elide-comments (`-e`) flag with `--format` (`-f`) `endoScript` or
+`endoZipBase64` (default) causes the bundler to blank out the interior of
+comments, without compromising line or column number cursor advancement.
+This can reduce bundle size without harming the debug experience any more than
+other transforms.
+
+Comment elision preserves `/*! slashasterbang /` comments and JSDoc comments
+with `@preserve`, `@copyright`, `@license` pragmas or the Internet Explorer
+`@cc_on` pragma.
+
+Comment elision does not strip comments entirely.
+The syntax to begin or end comments remains.
+
 ## Source maps
 
 With the `moduleFormat` of `endoZipBase64`, the bundler can generate source

packages/bundle-source/cache.js

@@ -19,6 +19,7 @@ const { Fail, quote: q } = assert;
  * @property {string} bundleTime ISO format
  * @property {number} bundleSize
  * @property {boolean} noTransforms
+ * @property {boolean} elideComments
  * @property {ModuleFormat} format
  * @property {string[]} conditions
  * @property {{ relative: string, absolute: string }} moduleSource
@@ -52,6 +53,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
    * @param {Logger} [log]
    * @param {object} [options]
    * @param {boolean} [options.noTransforms]
+   * @param {boolean} [options.elideComments]
    * @param {string[]} [options.conditions]
    * @param {ModuleFormat} [options.format]
    */
@@ -60,6 +62,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
 
     const {
       noTransforms = false,
+      elideComments = false,
       format = 'endoZipBase64',
       conditions = [],
     } = options;
@@ -93,7 +96,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
 
     const bundle = await bundleSource(
       rootPath,
-      { ...bundleOptions, noTransforms, format, conditions },
+      { ...bundleOptions, noTransforms, elideComments, format, conditions },
       {
         ...readPowers,
         read: loggedRead,
@@ -122,6 +125,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
         }),
       ),
       noTransforms,
+      elideComments,
       format,
       conditions,
     };
@@ -152,6 +156,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
    * @param {BundleMeta} [meta]
    * @param {object} [options]
    * @param {boolean} [options.noTransforms]
+   * @param {boolean} [options.elideComments]
    * @param {ModuleFormat} [options.format]
    * @param {string[]} [options.conditions]
    * @returns {Promise<BundleMeta>}
@@ -166,6 +171,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     await null;
     const {
       noTransforms: expectedNoTransforms = false,
+      elideComments: expectedElideComments = false,
       format: expectedFormat = DEFAULT_MODULE_FORMAT,
       conditions: expectedConditions = [],
     } = options;
@@ -194,12 +200,14 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
       moduleSource: { absolute: moduleSource },
       format = 'endoZipBase64',
       noTransforms = false,
+      elideComments = false,
       conditions = [],
     } = meta;
     conditions.sort();
     assert.equal(bundleFileName, toBundleName(targetName));
     assert.equal(format, expectedFormat);
     assert.equal(noTransforms, expectedNoTransforms);
+    assert.equal(elideComments, expectedElideComments);
     assert.equal(conditions.length, expectedConditions.length);
     conditions.forEach((tag, index) => {
       assert.equal(tag, expectedConditions[index]);
@@ -249,6 +257,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
    * @param {Logger} [log]
    * @param {object} [options]
    * @param {boolean} [options.noTransforms]
+   * @param {boolean} [options.elideComments]
    * @param {ModuleFormat} [options.format]
    * @param {string[]} [options.conditions]
    * @returns {Promise<BundleMeta>}
@@ -269,13 +278,15 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
         meta = await validate(targetName, rootPath, log, meta, {
           format: options.format,
           noTransforms: options.noTransforms,
+          elideComments: options.elideComments,
           conditions: options.conditions,
         });
         const {
           bundleTime,
           bundleSize,
           contents,
           noTransforms,
+          elideComments,
           format = 'endoZipBase64',
           conditions = [],
         } = meta;
@@ -288,7 +299,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
           bundleTime,
           'with size',
           bundleSize,
-          noTransforms ? 'w/o transforms' : 'with transforms',
+          `${noTransforms ? 'w/o transforms' : 'with transforms'}${elideComments ? ' and comments elided' : ''}`,
           'with format',
           format,
           'and conditions',
@@ -308,6 +319,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
         bundleTime,
         contents,
         noTransforms,
+        elideComments,
         format = 'endoZipBase64',
         conditions = [],
       } = meta;
@@ -319,7 +331,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
         bundleFileName,
         'at',
         bundleTime,
-        noTransforms ? 'w/o transforms' : 'with transforms',
+        `${noTransforms ? 'w/o transforms' : 'with transforms'}${elideComments ? ' and comments elided' : ''}`,
         'with format',
         format,
         'and conditions',
@@ -337,6 +349,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
    * @param {Logger} [log]
    * @param {object} [options]
    * @param {boolean} [options.noTransforms]
+   * @param {boolean} [options.elideComments]
    * @param {ModuleFormat} [options.format]
    * @param {string[]} [options.conditions]
    */

packages/bundle-source/demo/meaningful.js

@@ -1,3 +1,7 @@
 import json from './meaning.json';
 
+/**
+ * This comment will be blanked with the elideComments / --elide-comments
+ * feature enabled.
+ */
 export const meaning = json.meaning;

packages/bundle-source/src/endo.js

@@ -12,8 +12,14 @@ const textDecoder = new TextDecoder();
 
 export const makeBundlingKit = (
   { pathResolve, userInfo, computeSha512, platform, env },
-  { cacheSourceMaps, noTransforms, commonDependencies, dev },
+  { cacheSourceMaps, elideComments, noTransforms, commonDependencies, dev },
 ) => {
+  if (noTransforms && elideComments) {
+    throw new Error(
+      'bundleSource endoZipBase64 cannot elideComments with noTransforms',
+    );
+  }
+
   const sourceMapJobs = new Set();
   let writeSourceMap = Function.prototype;
   if (cacheSourceMaps) {
@@ -112,6 +118,7 @@ export const makeBundlingKit = (
       sourceType: babelSourceType,
       sourceMap,
       sourceMapUrl: new URL(specifier, location).href,
+      elideComments,
     }));
     const objectBytes = textEncoder.encode(object);
     return { bytes: objectBytes, parser, sourceMap };

packages/bundle-source/src/main.js

@@ -13,14 +13,20 @@ bundle-source [-Tft] --cache-js|--cache-json <cache/> (<entry.js> <bundle-name>)
   -f,--format endoZipBase64*|nestedEvaluate|getExport
   -C,--condition <condition> (browser, node, development, &c)
   -C development (to access devDependencies)
-  -T,--no-transforms`;
+  -T,--no-transforms
+  -e,--elide-comments`;
 
 const options = /** @type {const} */ ({
   'no-transforms': {
     type: 'boolean',
     short: 'T',
     multiple: false,
   },
+  'elide-comments': {
+    type: 'boolean',
+    short: 'e',
+    multiple: false,
+  },
   'cache-js': {
     type: 'string',
     multiple: false,
@@ -61,6 +67,7 @@ export const main = async (args, { loadModule, pid, log }) => {
       format: moduleFormat = 'endoZipBase64',
       condition: conditions = [],
       'no-transforms': noTransforms,
+      'elide-comments': elideComments,
       'cache-json': cacheJson,
       'cache-js': cacheJs,
       // deprecated
@@ -69,6 +76,11 @@ export const main = async (args, { loadModule, pid, log }) => {
     positionals,
   } = parseArgs({ args, options, allowPositionals: true });
 
+  if (noTransforms && elideComments) {
+    throw Error(
+      `Cannot elide comments without transforms (-T,--no-transforms + -e,--elide-comments)`,
+    );
+  }
   if (!SUPPORTED_FORMATS.includes(moduleFormat)) {
     throw Error(`Unsupported format: ${moduleFormat}\n\n${USAGE}`);
   }
@@ -94,6 +106,7 @@ export const main = async (args, { loadModule, pid, log }) => {
     const [entryPath] = positionals;
     const bundle = await bundleSource(entryPath, {
       noTransforms,
+      elideComments,
       format,
       conditions,
     });
@@ -125,6 +138,7 @@ export const main = async (args, { loadModule, pid, log }) => {
     // eslint-disable-next-line no-await-in-loop
     await cache.validateOrAdd(bundleRoot, bundleName, undefined, {
       noTransforms,
+      elideComments,
       format,
       conditions,
     });

packages/bundle-source/src/script.js

@@ -20,6 +20,7 @@ const readPowers = makeReadPowers({ fs, url, crypto });
  * @param {boolean} [options.dev]
  * @param {boolean} [options.cacheSourceMaps]
  * @param {boolean} [options.noTransforms]
+ * @param {boolean} [options.elideComments]
  * @param {Record<string, string>} [options.commonDependencies]
  * @param {object} [grantedPowers]
  * @param {(bytes: string | Uint8Array) => string} [grantedPowers.computeSha512]
@@ -37,9 +38,11 @@ export async function bundleScript(
     dev = false,
     cacheSourceMaps = false,
     noTransforms = false,
+    elideComments = false,
     conditions = [],
     commonDependencies,
   } = options;
+
   const powers = { ...readPowers, ...grantedPowers };
   const {
     computeSha512,
@@ -63,6 +66,7 @@ export async function bundleScript(
       {
         cacheSourceMaps,
         noTransforms,
+        elideComments,
         commonDependencies,
         dev,
       },

packages/bundle-source/src/types.js

@@ -67,6 +67,9 @@ export {};
  * @property {T} [format]
  * @property {boolean} [dev] - development mode, for test bundles that need
  * access to devDependencies of the entry package.
+ * @property {boolean} [elideComments] - when true for the `endoScript` and
+ * `endoZipBase64` format, replaces the interior of comments with blank space
+ * that advances the cursor the same number of lines and columns.
  * @property {boolean} [noTransforms] - when true, generates a bundle with the
  * original sources instead of SES-shim specific ESM and CJS. This may become
  * default in a future major version.

packages/bundle-source/src/zip-base64.js

@@ -22,6 +22,7 @@ const readPowers = makeReadPowers({ fs, url, crypto });
  * @param {boolean} [options.dev]
  * @param {boolean} [options.cacheSourceMaps]
  * @param {boolean} [options.noTransforms]
+ * @param {boolean} [options.elideComments]
  * @param {string[]} [options.conditions]
  * @param {Record<string, string>} [options.commonDependencies]
  * @param {object} [grantedPowers]
@@ -40,6 +41,7 @@ export async function bundleZipBase64(
     dev = false,
     cacheSourceMaps = false,
     noTransforms = false,
+    elideComments = false,
     conditions = [],
     commonDependencies,
   } = options;
@@ -66,6 +68,7 @@ export async function bundleZipBase64(
       {
         cacheSourceMaps,
         noTransforms,
+        elideComments,
         commonDependencies,
         dev,
       },

packages/bundle-source/test/endo-script-format.tests.js

@@ -4,16 +4,30 @@ import test from '@endo/ses-ava/prepare-endo.js';
 import * as url from 'url';
 import bundleSource from '../src/index.js';
 
-test('endo script format', async t => {
+const generate = async (options = {}) => {
   const entryPath = url.fileURLToPath(
     new URL(`../demo/meaning.js`, import.meta.url),
   );
-  const bundle = await bundleSource(entryPath, {
+  return bundleSource(entryPath, {
     format: 'endoScript',
+    ...options,
   });
+};
+
+test('endo script format', async t => {
+  const bundle = await generate();
   t.is(bundle.moduleFormat, 'endoScript');
   const { source } = bundle;
   const compartment = new Compartment();
   const ns = compartment.evaluate(source);
   t.is(ns.meaning, 42);
 });
+
+test('endo script format is smaller with blank comments', async t => {
+  const bigBundle = await generate();
+  const smallBundle = await generate({ elideComments: true });
+  const compartment = new Compartment();
+  const ns = compartment.evaluate(smallBundle.source);
+  t.is(ns.meaning, 42);
+  t.assert(smallBundle.source.length < bigBundle.source.length);
+});

packages/evasive-transform/NEWS.md

@@ -0,0 +1,6 @@
+User-visible changes in `@endo/evasive-transform`:
+
+# Next release
+
+- Adds an `elideComments` option to replace the interior of comments with
+  minimal blank space with identical cursor advancement behavior.

packages/evasive-transform/src/index.js

@@ -19,6 +19,7 @@ import { generate } from './generate.js';
  * @property {string|import('source-map').RawSourceMap} [sourceMap] - Original source map in JSON string or object form
  * @property {string} [sourceUrl] - URL or filepath of the original source in `code`
  * @property {boolean} [useLocationUnmap] - Enable location unmapping. Only applies if `sourceMap` was provided
+ * @property {boolean} [elideComments] - Replace comments with an ellipsis but preserve interior newlines.
  * @property {import('./parse-ast.js').SourceType} [sourceType] - Module source type
  * @public
  */
@@ -41,7 +42,13 @@ import { generate } from './generate.js';
 export function evadeCensorSync(source, options) {
   // TODO Use options ?? {} when resolved:
   // https://github.com/Agoric/agoric-sdk/issues/8671
-  const { sourceMap, sourceUrl, useLocationUnmap, sourceType } = options || {};
+  const {
+    sourceMap,
+    sourceUrl,
+    useLocationUnmap,
+    sourceType,
+    elideComments = false,
+  } = options || {};
 
   // Parse the rolled-up chunk with Babel.
   // We are prepared for different module systems.
@@ -53,9 +60,13 @@ export function evadeCensorSync(source, options) {
     typeof sourceMap === 'string' ? sourceMap : JSON.stringify(sourceMap);
 
   if (sourceMap && useLocationUnmap) {
-    transformAst(ast, { sourceMap: sourceMapJson, useLocationUnmap });
+    transformAst(ast, {
+      sourceMap: sourceMapJson,
+      useLocationUnmap,
+      elideComments,
+    });
   } else {
-    transformAst(ast);
+    transformAst(ast, { elideComments });
   }
 
   if (sourceUrl) {