diff --git a/packages/bundle-source/cache.js b/packages/bundle-source/cache.js index ddfa896419..4f380e769e 100644 --- a/packages/bundle-source/cache.js +++ b/packages/bundle-source/cache.js @@ -16,6 +16,7 @@ const { Fail, quote: q } = assert; * @property {string} bundleFileName * @property {string} bundleTime ISO format * @property {number} bundleSize + * @property {boolean} noTransforms * @property {{ relative: string, absolute: string }} moduleSource * @property {Array<{ relativePath: string, mtime: string, size: number }>} contents */ @@ -41,9 +42,18 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { ...bundleOptions } = opts || {}; - const add = async (rootPath, targetName, log = defaultLog) => { + /** + * @param {string} rootPath + * @param {string} targetName + * @param {Logger} [log] + * @param {object} [options] + * @param {boolean} [options.noTransforms] + */ + const add = async (rootPath, targetName, log = defaultLog, options = {}) => { const srcRd = cwd.neighbor(rootPath); + const { noTransforms = false } = options; + const statsByPath = new Map(); const loggedRead = async loc => { @@ -69,10 +79,14 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { const bundleWr = wr.neighbor(bundleFileName); const metaWr = wr.neighbor(toBundleMeta(targetName)); - const bundle = await bundleSource(rootPath, bundleOptions, { - ...readPowers, - read: loggedRead, - }); + const bundle = await bundleSource( + rootPath, + { ...bundleOptions, noTransforms }, + { + ...readPowers, + read: loggedRead, + }, + ); const { moduleFormat } = bundle; assert.equal(moduleFormat, 'endoZipBase64'); @@ -98,6 +112,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { size, }), ), + noTransforms, }; await metaWr.atomicWriteText(JSON.stringify(meta, null, 2)); @@ -200,9 +215,16 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { * @param {string} rootPath * @param {string} targetName * @param {Logger} [log] + * @param {object} [options] + * @param {boolean} [options.noTransforms] * @returns {Promise} */ - const validateOrAdd = async (rootPath, targetName, log = defaultLog) => { + const validateOrAdd = async ( + rootPath, + targetName, + log = defaultLog, + options = {}, + ) => { const metaText = await loadMetaText(targetName, log); /** @type {BundleMeta | undefined} */ @@ -211,7 +233,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { if (meta !== undefined) { try { meta = await validate(targetName, rootPath, log, meta); - const { bundleTime, bundleSize, contents } = meta; + const { bundleTime, bundleSize, contents, noTransforms } = meta; log( `${wr}`, toBundleName(targetName), @@ -221,6 +243,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { bundleTime, 'with size', bundleSize, + noTransforms ? 'w/o transforms' : 'with transforms', ); } catch (invalid) { meta = undefined; @@ -230,8 +253,8 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { if (meta === undefined) { log(`${wr}`, 'add:', targetName, 'from', rootPath); - meta = await add(rootPath, targetName, log); - const { bundleFileName, bundleTime, contents } = meta; + meta = await add(rootPath, targetName, log, options); + const { bundleFileName, bundleTime, contents, noTransforms } = meta; log( `${wr}`, 'bundled', @@ -240,6 +263,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { bundleFileName, 'at', bundleTime, + noTransforms ? 'w/o transforms' : 'with transforms', ); } @@ -251,11 +275,14 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { * @param {string} rootPath * @param {string} [targetName] * @param {Logger} [log] + * @param {object} [options] + * @param {boolean} [options.noTransforms] */ const load = async ( rootPath, targetName = readPowers.basename(rootPath, '.js'), log = defaultLog, + options = {}, ) => { const found = loaded.get(targetName); // console.log('load', { targetName, found: !!found, rootPath }); @@ -264,7 +291,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => { } const todo = makePromiseKit(); loaded.set(targetName, { rootPath, bundle: todo.promise }); - const bundle = await validateOrAdd(rootPath, targetName, log) + const bundle = await validateOrAdd(rootPath, targetName, log, options) .then( ({ bundleFileName }) => import(`${wr.readOnly().neighbor(bundleFileName)}`), @@ -298,12 +325,12 @@ export const makeNodeBundleCache = async ( nonce, ) => { const [fs, path, url, crypto, timers, os] = await Promise.all([ - await loadModule('fs'), - await loadModule('path'), - await loadModule('url'), - await loadModule('crypto'), - await loadModule('timers'), - await loadModule('os'), + loadModule('fs'), + loadModule('path'), + loadModule('url'), + loadModule('crypto'), + loadModule('timers'), + loadModule('os'), ]); if (nonce === undefined) { diff --git a/packages/bundle-source/package.json b/packages/bundle-source/package.json index 93b401906e..a867ab6a45 100644 --- a/packages/bundle-source/package.json +++ b/packages/bundle-source/package.json @@ -39,8 +39,10 @@ "devDependencies": { "@endo/lockdown": "^1.0.7", "@endo/ses-ava": "^1.2.2", + "@endo/zip": "^1.0.5", "ava": "^6.1.3", "c8": "^7.14.0", + "eslint": "^8.57.0", "typescript": "5.5.0-beta" }, "keywords": [], diff --git a/packages/bundle-source/src/main.js b/packages/bundle-source/src/main.js index 813d1194e1..8115ffe574 100644 --- a/packages/bundle-source/src/main.js +++ b/packages/bundle-source/src/main.js @@ -1,9 +1,31 @@ // @ts-check +import { parseArgs } from 'util'; import { jsOpts, jsonOpts, makeNodeBundleCache } from '../cache.js'; const USAGE = 'bundle-source [--cache-js | --cache-json] cache/ module1.js bundleName1 module2.js bundleName2 ...'; +const options = /** @type {const} */ ({ + 'no-transforms': { + type: 'boolean', + short: 'T', + multiple: false, + }, + 'cache-js': { + type: 'string', + multiple: false, + }, + 'cache-json': { + type: 'string', + multiple: false, + }, + // deprecated + to: { + type: 'string', + multiple: false, + }, +}); + /** * @param {[to: string, dest: string, ...rest: string[]]} args * @param {object} powers @@ -13,20 +35,42 @@ const USAGE = * @returns {Promise} */ export const main = async (args, { loadModule, pid, log }) => { - const [to, dest, ...pairs] = args; - if (!(dest && pairs.length > 0 && pairs.length % 2 === 0)) { + const { + values: { + 'no-transforms': noTransforms, + 'cache-json': cacheJson, + 'cache-js': cacheJs, + // deprecated + to: cacheJsAlias, + }, + positionals: pairs, + } = parseArgs({ args, options, allowPositionals: true }); + + if ( + !( + pairs.length > 0 && + pairs.length % 2 === 0 && + [cacheJson, cacheJs, cacheJsAlias].filter(Boolean).length === 1 + ) + ) { throw Error(USAGE); } + /** @type {string} */ + let dest; let cacheOpts; // `--to` option is deprecated, but we now use it to mean `--cache-js`. - if (to === '--to') { + if (cacheJs !== undefined) { + dest = cacheJs; cacheOpts = jsOpts; - } else if (to === '--cache-js') { + } else if (cacheJsAlias !== undefined) { + dest = cacheJsAlias; cacheOpts = jsOpts; - } else if (to === '--cache-json') { + } else if (cacheJson !== undefined) { + dest = cacheJson; cacheOpts = jsonOpts; } else { + // unreachable throw Error(USAGE); } @@ -41,6 +85,8 @@ export const main = async (args, { loadModule, pid, log }) => { const [bundleRoot, bundleName] = pairs.slice(ix, ix + 2); // eslint-disable-next-line no-await-in-loop - await cache.validateOrAdd(bundleRoot, bundleName); + await cache.validateOrAdd(bundleRoot, bundleName, undefined, { + noTransforms, + }); } }; diff --git a/packages/bundle-source/src/types.js b/packages/bundle-source/src/types.js index 94d3ac881c..07bb8d4df2 100644 --- a/packages/bundle-source/src/types.js +++ b/packages/bundle-source/src/types.js @@ -64,6 +64,9 @@ export {}; * @property {T} [format] * @property {boolean} [dev] - development mode, for test bundles that need * access to devDependencies of the entry package. + * @property {boolean} [noTransforms] - when true, generates a bundle with the + * original sources instead of SES-shim specific ESM and CJS. This may become + * default in a future major version. */ /** diff --git a/packages/bundle-source/src/zip-base64.js b/packages/bundle-source/src/zip-base64.js index b81102f497..81a903c45d 100644 --- a/packages/bundle-source/src/zip-base64.js +++ b/packages/bundle-source/src/zip-base64.js @@ -7,7 +7,10 @@ import url from 'url'; import fs from 'fs'; import os from 'os'; -import { makeAndHashArchive } from '@endo/compartment-mapper/archive.js'; +import { defaultParserForLanguage as transformingParserForLanguage } from '@endo/compartment-mapper/archive-parsers.js'; +import { defaultParserForLanguage as transparentParserForLanguage } from '@endo/compartment-mapper/import-parsers.js'; +import { mapNodeModules } from '@endo/compartment-mapper/node-modules.js'; +import { makeAndHashArchiveFromMap } from '@endo/compartment-mapper/archive-lite.js'; import { encodeBase64 } from '@endo/base64'; import { whereEndoCache } from '@endo/where'; import { makeReadPowers } from '@endo/compartment-mapper/node-powers.js'; @@ -17,12 +20,31 @@ const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); const readPowers = makeReadPowers({ fs, url, crypto }); +/** + * @param {string} startFilename + * @param {object} [options] + * @param {boolean} [options.dev] + * @param {boolean} [options.cacheSourceMaps] + * @param {boolean} [options.noTransforms] + * @param {Record} [options.commonDependencies] + * @param {object} [grantedPowers] + * @param {(bytes: string | Uint8Array) => string} [grantedPowers.computeSha512] + * @param {typeof import('path)['resolve']} [grantedPowers.pathResolve] + * @param {typeof import('os')['userInfo']} [grantedPowers.userInfo] + * @param {typeof process['env']} [grantedPowers.env] + * @param {typeof process['platform']} [grantedPowers.platform] + */ export async function bundleZipBase64( startFilename, options = {}, grantedPowers = {}, ) { - const { dev = false, cacheSourceMaps = false, commonDependencies } = options; + const { + dev = false, + cacheSourceMaps = false, + noTransforms = false, + commonDependencies, + } = options; const powers = { ...readPowers, ...grantedPowers }; const { computeSha512, @@ -137,9 +159,11 @@ export async function bundleZipBase64( return { bytes: objectBytes, parser, sourceMap }; }; - const { bytes, sha512 } = await makeAndHashArchive(powers, entry, { - dev, - moduleTransforms: { + let parserForLanguage = transparentParserForLanguage; + let moduleTransforms = {}; + if (!noTransforms) { + parserForLanguage = transformingParserForLanguage; + moduleTransforms = { async mjs( sourceBytes, specifier, @@ -170,12 +194,25 @@ export async function bundleZipBase64( sourceMap, ); }, - }, - sourceMapHook(sourceMap, sourceDescriptor) { - sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor)); - }, + }; + } + + const compartmentMap = await mapNodeModules(powers, entry, { + dev, commonDependencies, }); + + const { bytes, sha512 } = await makeAndHashArchiveFromMap( + powers, + compartmentMap, + { + parserForLanguage, + moduleTransforms, + sourceMapHook(sourceMap, sourceDescriptor) { + sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor)); + }, + }, + ); assert(sha512); await Promise.all(sourceMapJobs); const endoZipBase64 = encodeBase64(bytes); diff --git a/packages/bundle-source/test/no-transforms.test.js b/packages/bundle-source/test/no-transforms.test.js new file mode 100644 index 0000000000..6079534b6b --- /dev/null +++ b/packages/bundle-source/test/no-transforms.test.js @@ -0,0 +1,36 @@ +// @ts-check +import test from '@endo/ses-ava/prepare-endo.js'; + +import fs from 'fs'; +import url from 'url'; +import { decodeBase64 } from '@endo/base64'; +import { ZipReader } from '@endo/zip'; +import bundleSource from '../src/index.js'; + +test('no-transforms applies no transforms', async t => { + const entryPath = url.fileURLToPath( + new URL(`../demo/circular/a.js`, import.meta.url), + ); + const { endoZipBase64 } = await bundleSource(entryPath, { + moduleFormat: 'endoZipBase64', + noTransforms: true, + }); + const endoZipBytes = decodeBase64(endoZipBase64); + const zipReader = new ZipReader(endoZipBytes); + const compartmentMapBytes = zipReader.read('compartment-map.json'); + const compartmentMapText = new TextDecoder().decode(compartmentMapBytes); + const compartmentMap = JSON.parse(compartmentMapText); + const { entry, compartments } = compartmentMap; + const compartment = compartments[entry.compartment]; + const module = compartment.modules[entry.module]; + // Alleged module type is not precompiled (pre-mjs-json) + t.is(module.parser, 'mjs'); + + const moduleBytes = zipReader.read( + `${compartment.location}/${module.location}`, + ); + const moduleText = new TextDecoder().decode(moduleBytes); + const originalModuleText = await fs.promises.readFile(entryPath, 'utf-8'); + // And, just to be sure, the text in the bundle matches the original text. + t.is(moduleText, originalModuleText); +}); diff --git a/yarn.lock b/yarn.lock index 847133a331..761dcfd3bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -190,11 +190,13 @@ __metadata: "@endo/promise-kit": "npm:^1.1.2" "@endo/ses-ava": "npm:^1.2.2" "@endo/where": "npm:^1.0.5" + "@endo/zip": "npm:^1.0.5" "@rollup/plugin-commonjs": "npm:^19.0.0" "@rollup/plugin-node-resolve": "npm:^13.0.0" acorn: "npm:^8.2.4" ava: "npm:^6.1.3" c8: "npm:^7.14.0" + eslint: "npm:^8.57.0" rollup: "npm:^2.79.1" typescript: "npm:5.5.0-beta" bin: