diff --git a/packages/SwingSet/misc-tools/replay-transcript.js b/packages/SwingSet/misc-tools/replay-transcript.js index 623b8d2e47d..5e658031d67 100644 --- a/packages/SwingSet/misc-tools/replay-transcript.js +++ b/packages/SwingSet/misc-tools/replay-transcript.js @@ -543,6 +543,7 @@ async function replay(transcriptFile) { const { hash, compressSeconds: saveSeconds } = await manager.makeSnapshot( lastTranscriptNum, snapStore, + false, // Do not restart, we'll do that ourselves if needed ); fs.writeSync( snapshotActivityFd, diff --git a/packages/SwingSet/src/controller/controller.js b/packages/SwingSet/src/controller/controller.js index 12d2d940530..b47d1ce58e3 100644 --- a/packages/SwingSet/src/controller/controller.js +++ b/packages/SwingSet/src/controller/controller.js @@ -435,7 +435,7 @@ export async function makeSwingsetController( * slogCallbacks?: unknown; * slogSender?: import('@agoric/telemetry').SlogSender; * testTrackDecref?: unknown; - * warehousePolicy?: { maxVatsOnline?: number }; + * warehousePolicy?: import('../types-external.js').VatWarehousePolicy; * }} runtimeOptions * @param {Record} deviceEndowments * @typedef { import('@agoric/swing-store').KVStore } KVStore diff --git a/packages/SwingSet/src/controller/startXSnap.js b/packages/SwingSet/src/controller/startXSnap.js index 82bf48bbbc9..e9d1c3649bd 100644 --- a/packages/SwingSet/src/controller/startXSnap.js +++ b/packages/SwingSet/src/controller/startXSnap.js @@ -5,6 +5,32 @@ import { xsnap, recordXSnap } from '@agoric/xsnap'; const NETSTRING_MAX_CHUNK_SIZE = 12_000_000; +/** + * @typedef {object} StartXSnapInitFromBundlesDetails + * @property {'bundles'} from + * + * TODO: Move bundleIDs here + */ + +/** + * @typedef {object} StartXSnapInitFromSnapshotStreamDetails + * @property {'snapshotStream'} from + * @property {AsyncIterable} snapshotStream + * @property {string} [snapshotDescription] + */ + +/** + * @typedef {object} StartXSnapInitFromSnapStoreDetails + * @property {'snapStore'} from + * @property {string} vatID + * + * TODO: transition to direct snapshot stream, and remove this option + */ + +/** @typedef {StartXSnapInitFromBundlesDetails | StartXSnapInitFromSnapshotStreamDetails | StartXSnapInitFromSnapStoreDetails} StartXSnapInitDetails */ + +/** @typedef {ReturnType} StartXSnap */ + /** * @param {{ * bundleHandler: import('./bundle-handler.js').BundleHandler, @@ -62,32 +88,62 @@ export function makeStartXSnap(options) { netstringMaxChunkSize: NETSTRING_MAX_CHUNK_SIZE, }; + /** @param {StartXSnapInitDetails} initDetails */ + function getSnapshotLoadOptions(initDetails) { + switch (initDetails.from) { + case 'bundles': + return undefined; + + case 'snapStore': { + if (!snapStore) { + // Fallback to load from bundles + return undefined; + } + + const { vatID } = initDetails; + const { hash: snapshotID, snapPos } = snapStore.getSnapshotInfo(vatID); + // console.log('startXSnap from', { snapshotID }); + const snapshotStream = snapStore.loadSnapshot(vatID); + const snapshotDescription = `${vatID}-${snapPos}-${snapshotID}`; + return { snapshotStream, snapshotDescription }; + } + + case 'snapshotStream': { + const { snapshotStream, snapshotDescription } = initDetails; + return { snapshotStream, snapshotDescription }; + } + + default: + // @ts-expect-error exhaustive check + throw Fail`Unexpected xsnap init type ${initDetails.type}`; + } + } + /** - * @param {string} vatID * @param {string} name * @param {object} details * @param {import('../types-external.js').BundleID[]} details.bundleIDs * @param {(request: Uint8Array) => Promise} details.handleCommand * @param {boolean} [details.metered] - * @param {boolean} [details.reload] + * @param {StartXSnapInitDetails} [details.init] */ async function startXSnap( - vatID, name, - { bundleIDs, handleCommand, metered, reload = false }, + { + bundleIDs, + handleCommand, + metered, + init: initDetails = { from: 'bundles' }, + }, ) { const meterOpts = metered ? {} : { meteringLimit: 0 }; - if (snapStore && reload) { - const { hash: snapshotID, snapPos } = snapStore.getSnapshotInfo(vatID); - // console.log('startXSnap from', { snapshotID }); - const snapshotStream = snapStore.loadSnapshot(vatID); + const snapshotLoadOpts = getSnapshotLoadOptions(initDetails); + if (snapshotLoadOpts) { // eslint-disable-next-line @jessie.js/no-nested-await const xs = await doXSnap({ - snapshotStream, - // TODO - snapshotDescription: `${vatID}-${snapPos}-${snapshotID}`, name, handleCommand, + ...snapshotLoadOpts, ...meterOpts, ...xsnapOpts, }); diff --git a/packages/SwingSet/src/kernel/state/vatKeeper.js b/packages/SwingSet/src/kernel/state/vatKeeper.js index 7b51a3cd8f4..fe15e7d746e 100644 --- a/packages/SwingSet/src/kernel/state/vatKeeper.js +++ b/packages/SwingSet/src/kernel/state/vatKeeper.js @@ -547,16 +547,21 @@ export function makeVatKeeper( * Store a snapshot, if given a snapStore. * * @param {VatManager} manager + * @param {boolean} [restartWorker] * @returns {Promise} */ - async function saveSnapshot(manager) { + async function saveSnapshot(manager, restartWorker) { if (!snapStore || !manager.makeSnapshot) { return; } // tell the manager to save a heap snapshot to the snapStore const endPosition = getTranscriptEndPosition(); - const info = await manager.makeSnapshot(endPosition, snapStore); + const info = await manager.makeSnapshot( + endPosition, + snapStore, + restartWorker, + ); const { hash: snapshotID, @@ -586,6 +591,7 @@ export function makeVatKeeper( compressedSize, compressSeconds, endPosition, + restartWorker, }); } diff --git a/packages/SwingSet/src/kernel/vat-loader/manager-helper.js b/packages/SwingSet/src/kernel/vat-loader/manager-helper.js index 157f76d74c0..33f2e07df98 100644 --- a/packages/SwingSet/src/kernel/vat-loader/manager-helper.js +++ b/packages/SwingSet/src/kernel/vat-loader/manager-helper.js @@ -11,6 +11,7 @@ import { * @typedef {import('@agoric/swingset-liveslots').VatSyscallObject} VatSyscallObject * @typedef {import('@agoric/swingset-liveslots').VatSyscallResult} VatSyscallResult * @typedef {import('../../types-internal.js').VatManager} VatManager + * @typedef {import('../../types-internal.js').MakeSnapshot} MakeSnapshot */ // We use vat-centric terminology here, so "inbound" means "into a vat", @@ -57,7 +58,7 @@ import { /** * * @typedef { { getManager: (shutdown: () => Promise, - * makeSnapshot?: (snapPos: number, ss: SnapStore) => Promise) => VatManager, + * makeSnapshot?: MakeSnapshot) => VatManager, * syscallFromWorker: (vso: VatSyscallObject) => VatSyscallResult, * setDeliverToWorker: (dtw: unknown) => void, * } } ManagerKit @@ -170,7 +171,7 @@ function makeManagerKit(retainSyscall = false) { /** * * @param { () => Promise} shutdown - * @param {(snapPos: number, ss: SnapStore) => Promise} [makeSnapshot] + * @param {MakeSnapshot} [makeSnapshot] * @returns {VatManager} */ function getManager(shutdown, makeSnapshot) { diff --git a/packages/SwingSet/src/kernel/vat-loader/manager-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vat-loader/manager-subprocess-xsnap.js index ac80d870e47..e22cbed4478 100644 --- a/packages/SwingSet/src/kernel/vat-loader/manager-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vat-loader/manager-subprocess-xsnap.js @@ -1,3 +1,4 @@ +import { synchronizedTee } from '@agoric/internal'; import { assert, Fail, q } from '@agoric/assert'; import { ExitCode } from '@agoric/xsnap/api.js'; import { makeManagerKit } from './manager-helper.js'; @@ -26,16 +27,31 @@ function parentLog(first, ...args) { const encoder = new TextEncoder(); const decoder = new TextDecoder(); +/** @param { (item: Tagged) => unknown } [handleUpstream] */ +const makeRevokableHandleCommandKit = handleUpstream => { + /** + * @param {Uint8Array} msg + * @returns {Promise} + */ + const handleCommand = async msg => { + // parentLog('handleCommand', { length: msg.byteLength }); + if (!handleUpstream) { + throw Fail`Worker received command after revocation`; + } + const tagged = handleUpstream(JSON.parse(decoder.decode(msg))); + return encoder.encode(JSON.stringify(tagged)); + }; + const revoke = () => { + handleUpstream = undefined; + }; + return harden({ handleCommand, revoke }); +}; + /** * @param {{ * kernelKeeper: KernelKeeper, * kernelSlog: KernelSlog, - * startXSnap: (vatID: string, name: string, - * details: { - * bundleIDs: BundleID[], - * handleCommand: AsyncHandler, - * metered?: boolean, - * reload?: boolean } ) => Promise, + * startXSnap: import('../../controller/startXSnap.js').StartXSnap, * testLog: (...args: unknown[]) => void, * }} tools * @returns {VatManagerFactory} @@ -110,13 +126,6 @@ export function makeXsSubprocessFactory({ } } - /** @type { (msg: Uint8Array) => Promise } */ - async function handleCommand(msg) { - // parentLog('handleCommand', { length: msg.byteLength }); - const tagged = handleUpstream(JSON.parse(decoder.decode(msg))); - return encoder.encode(JSON.stringify(tagged)); - } - const vatKeeper = kernelKeeper.provideVatKeeper(vatID); const snapshotInfo = vatKeeper.getSnapshotInfo(); if (snapshotInfo) { @@ -128,12 +137,15 @@ export function makeXsSubprocessFactory({ // a shell-escape attack const argName = `${vatID}:${vatName !== undefined ? vatName : ''}`; + /** @type {ReturnType | undefined} */ + let handleCommandKit = makeRevokableHandleCommandKit(handleUpstream); + // start the worker and establish a connection - const worker = await startXSnap(vatID, argName, { + let worker = await startXSnap(argName, { bundleIDs, - handleCommand, + handleCommand: handleCommandKit.handleCommand, metered, - reload: !!snapshotInfo, + init: snapshotInfo && { from: 'snapStore', vatID }, }); /** @type { (item: Tagged) => Promise } */ @@ -215,19 +227,63 @@ export function makeXsSubprocessFactory({ mk.setDeliverToWorker(deliverToWorker); function shutdown() { + handleCommandKit?.revoke(); + handleCommandKit = undefined; return worker.close().then(_ => undefined); } /** * @param {number} snapPos * @param {SnapStore} snapStore + * @param {boolean} [restartWorker] * @returns {Promise} */ - function makeSnapshot(snapPos, snapStore) { - return snapStore.saveSnapshot( - vatID, - snapPos, - worker.makeSnapshotStream(`${vatID}-${snapPos}`), + async function makeSnapshot(snapPos, snapStore, restartWorker) { + const snapshotDescription = `${vatID}-${snapPos}`; + const snapshotStream = worker.makeSnapshotStream(snapshotDescription); + + if (!restartWorker) { + return snapStore.saveSnapshot(vatID, snapPos, snapshotStream); + } + + /** @type {AsyncGenerator[]} */ + const [restartWorkerStream, snapStoreSaveStream] = synchronizedTee( + snapshotStream, + 2, ); + + handleCommandKit?.revoke(); + handleCommandKit = makeRevokableHandleCommandKit(handleUpstream); + + let snapshotResults; + const closeP = worker.close(); + [worker, snapshotResults] = await Promise.all([ + startXSnap(argName, { + bundleIDs, + handleCommand: handleCommandKit.handleCommand, + metered, + init: { + from: 'snapshotStream', + snapshotStream: restartWorkerStream, + snapshotDescription, + }, + }), + snapStore.saveSnapshot(vatID, snapPos, snapStoreSaveStream), + ]); + await closeP; + + /** @type {Partial} */ + const reloadSnapshotInfo = { + snapPos, + hash: snapshotResults.hash, + }; + + kernelSlog.write({ + type: 'heap-snapshot-load', + vatID, + ...reloadSnapshotInfo, + }); + + return snapshotResults; } return mk.getManager(shutdown, makeSnapshot); diff --git a/packages/SwingSet/src/kernel/vat-warehouse.js b/packages/SwingSet/src/kernel/vat-warehouse.js index b78cc7a5eb0..ea2633f3106 100644 --- a/packages/SwingSet/src/kernel/vat-warehouse.js +++ b/packages/SwingSet/src/kernel/vat-warehouse.js @@ -243,7 +243,8 @@ export function makeVatWarehouse({ panic, warehousePolicy, }) { - const { maxVatsOnline = 50 } = warehousePolicy || {}; + const { maxVatsOnline = 50, restartWorkerOnSnapshot = true } = + warehousePolicy || {}; // Often a large contract evaluation is among the first few deliveries, // so let's do a snapshot after just a few deliveries. const snapshotInitial = kernelKeeper.getSnapshotInitial(); @@ -603,8 +604,10 @@ export function makeVatWarehouse({ // vatKeeper.saveSnapshot() pushes a save-snapshot transcript // entry, then starts a new transcript span, then pushes a // load-snapshot entry, so that the current span always starts - // with an initialize-snapshot or load-snapshot pseudo-delivery - await vatKeeper.saveSnapshot(manager); + // with an initialize-snapshot or load-snapshot pseudo-delivery, + // regardless of whether the worker was restarted from snapshot + // or not. + await vatKeeper.saveSnapshot(manager, restartWorkerOnSnapshot); return true; } diff --git a/packages/SwingSet/src/types-external.js b/packages/SwingSet/src/types-external.js index 76d15bffd85..382fb425622 100644 --- a/packages/SwingSet/src/types-external.js +++ b/packages/SwingSet/src/types-external.js @@ -236,6 +236,7 @@ export {}; * * @typedef {object} VatWarehousePolicy * @property { number } [maxVatsOnline] Limit the number of simultaneous workers + * @property { boolean } [restartWorkerOnSnapshot] Reload worker immediately upon snapshot creation */ /** diff --git a/packages/SwingSet/src/types-internal.js b/packages/SwingSet/src/types-internal.js index e57977ad8c6..e40391d33c5 100644 --- a/packages/SwingSet/src/types-internal.js +++ b/packages/SwingSet/src/types-internal.js @@ -57,9 +57,11 @@ export {}; * bundle?: Bundle, * }} ManagerOptions * + * @typedef {(snapPos: number, ss: SnapStore, restartWorker?: boolean) => Promise} MakeSnapshot + * * @typedef { { deliver: (delivery: VatDeliveryObject, vatSyscallHandler: VatSyscallHandler) * => Promise, - * makeSnapshot?: undefined | ((snapPos: number, ss: SnapStore) => Promise), + * makeSnapshot?: undefined | MakeSnapshot, * shutdown: () => Promise, * } } VatManager * @typedef { { createFromBundle: (vatID: string, diff --git a/packages/SwingSet/test/test-controller.js b/packages/SwingSet/test/test-controller.js index 892041a6d45..545cb182ab5 100644 --- a/packages/SwingSet/test/test-controller.js +++ b/packages/SwingSet/test/test-controller.js @@ -255,6 +255,9 @@ test('static vats are unmetered on XS', async t => { limited.push(args.includes('-l')); return spawn(command, args, options); }, + warehousePolicy: { + restartWorkerOnSnapshot: false, + }, }, ); t.teardown(c.shutdown); diff --git a/packages/SwingSet/test/test-xsnap-metering.js b/packages/SwingSet/test/test-xsnap-metering.js index d706bf5bf82..33dd057903f 100644 --- a/packages/SwingSet/test/test-xsnap-metering.js +++ b/packages/SwingSet/test/test-xsnap-metering.js @@ -52,11 +52,10 @@ async function doTest(t, metered) { const store = makeSnapStore(db, () => {}, makeSnapStoreIO()); const { p: p1, startXSnap: start1 } = make(store); - const worker1 = await start1('vat', 'name', { + const worker1 = await start1('name', { bundleIDs: [], handleCommand, metered, - reload: false, }); const spawnArgs1 = await p1; checkMetered(t, spawnArgs1, metered); @@ -68,11 +67,11 @@ async function doTest(t, metered) { // and load it into a new worker const { p: p2, startXSnap: start2 } = make(store); - const worker2 = await start2('vat', 'name', { + const worker2 = await start2('name', { bundleIDs: [], handleCommand, metered, - reload: true, + init: { from: 'snapStore', vatID: 'vat' }, }); const spawnArgs2 = await p2; checkMetered(t, spawnArgs2, metered); diff --git a/packages/SwingSet/test/vat-warehouse/test-reload-snapshot.js b/packages/SwingSet/test/vat-warehouse/test-reload-snapshot.js index c09778a2830..0baf741f2d1 100644 --- a/packages/SwingSet/test/vat-warehouse/test-reload-snapshot.js +++ b/packages/SwingSet/test/vat-warehouse/test-reload-snapshot.js @@ -9,7 +9,7 @@ import { } from '@agoric/swing-store'; import { initializeSwingset, makeSwingsetController } from '../../src/index.js'; -test('vat reload from snapshot', async t => { +const vatReloadFromSnapshot = async (t, restartWorkerOnSnapshot) => { const config = { defaultReapInterval: 'never', snapshotInitial: 3, @@ -30,7 +30,10 @@ test('vat reload from snapshot', async t => { const argv = []; await initializeSwingset(config, argv, kernelStorage); - const c1 = await makeSwingsetController(kernelStorage, null); + const warehousePolicy = { restartWorkerOnSnapshot }; + const runtimeOptions = { warehousePolicy }; + + const c1 = await makeSwingsetController(kernelStorage, null, runtimeOptions); c1.pinVatRoot('target'); const vatID = c1.vatNameToID('target'); @@ -97,4 +100,7 @@ test('vat reload from snapshot', async t => { t.deepEqual(c2.dump().log, expected2); // note: *not* 0-11 t.deepEqual(getPositions(), [18, 19, 23]); await c2.shutdown(); -}); +}; + +test('vat reload from snapshot (restart worker)', vatReloadFromSnapshot, true); +test('vat reload from snapshot (reuse worker)', vatReloadFromSnapshot, false); diff --git a/packages/swing-store/src/snapStore.js b/packages/swing-store/src/snapStore.js index cdef408bb39..b3862fa0784 100644 --- a/packages/swing-store/src/snapStore.js +++ b/packages/swing-store/src/snapStore.js @@ -29,7 +29,7 @@ import { buffer } from './util.js'; * * @typedef {{ * loadSnapshot: (vatID: string) => AsyncIterableIterator, - * saveSnapshot: (vatID: string, snapPos: number, snapshotStream: AsyncIterableIterator) => Promise, + * saveSnapshot: (vatID: string, snapPos: number, snapshotStream: AsyncIterable) => Promise, * deleteAllUnusedSnapshots: () => void, * deleteVatSnapshots: (vatID: string) => void, * stopUsingLastSnapshot: (vatID: string) => void, @@ -167,7 +167,7 @@ export function makeSnapStore( * * @param {string} vatID * @param {number} snapPos - * @param {AsyncIterableIterator} snapshotStream + * @param {AsyncIterable} snapshotStream * @returns {Promise} */ async function saveSnapshot(vatID, snapPos, snapshotStream) {