From a297102b91afba4f523a442b0e4a15e479192b01 Mon Sep 17 00:00:00 2001 From: Chip Morningstar Date: Wed, 8 Mar 2023 13:58:06 -0800 Subject: [PATCH] chore: revisions based on review feedback --- .../SwingSet/src/kernel/state/kernelKeeper.js | 2 +- .../SwingSet/src/kernel/state/vatKeeper.js | 10 +- packages/SwingSet/src/kernel/vat-warehouse.js | 2 +- .../test/device-plugin/test-device.js | 6 + .../vat-admin/terminate/test-terminate.js | 10 +- packages/swing-store/src/snapStore.js | 128 +++-- packages/swing-store/src/swingStore.js | 46 +- packages/swing-store/src/transcriptStore.js | 113 ++++- .../swing-store/test/test-exportImport.js | 442 ++++++++++++++++++ packages/swing-store/test/test-snapstore.js | 1 + packages/swing-store/test/test-state.js | 1 + packages/swing-store/test/test-stateSync.js | 232 --------- 12 files changed, 689 insertions(+), 304 deletions(-) create mode 100644 packages/swing-store/test/test-exportImport.js delete mode 100644 packages/swing-store/test/test-stateSync.js diff --git a/packages/SwingSet/src/kernel/state/kernelKeeper.js b/packages/SwingSet/src/kernel/state/kernelKeeper.js index 81762156f1f..8dda7e69f56 100644 --- a/packages/SwingSet/src/kernel/state/kernelKeeper.js +++ b/packages/SwingSet/src/kernel/state/kernelKeeper.js @@ -784,7 +784,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { const promisePrefix = `${vatID}.c.p`; const kernelPromisesToReject = []; - vatKeeper.deleteSnapshots(); + vatKeeper.removeSnapshotAndTranscript(); // Note: ASCII order is "+,-./", and we rely upon this to split the // keyspace into the various o+NN/o-NN/etc spaces. If we were using a diff --git a/packages/SwingSet/src/kernel/state/vatKeeper.js b/packages/SwingSet/src/kernel/state/vatKeeper.js index a66f40d8e93..5da8446e981 100644 --- a/packages/SwingSet/src/kernel/state/vatKeeper.js +++ b/packages/SwingSet/src/kernel/state/vatKeeper.js @@ -549,9 +549,12 @@ export function makeVatKeeper( } function removeSnapshotAndTranscript() { - if (snapStore) { - snapStore.deleteVatSnapshots(vatID); - } + deleteSnapshots(); + transcriptStore.deleteVatTranscripts(vatID); + } + + function removeSnapshotAndResetTranscript() { + deleteSnapshots(); transcriptStore.rolloverSpan(vatID); } @@ -627,5 +630,6 @@ export function makeVatKeeper( deleteSnapshots, getSnapshotInfo, removeSnapshotAndTranscript, + removeSnapshotAndResetTranscript, }); } diff --git a/packages/SwingSet/src/kernel/vat-warehouse.js b/packages/SwingSet/src/kernel/vat-warehouse.js index 7c5ecbf888a..d96b485238f 100644 --- a/packages/SwingSet/src/kernel/vat-warehouse.js +++ b/packages/SwingSet/src/kernel/vat-warehouse.js @@ -382,7 +382,7 @@ export function makeVatWarehouse(kernelKeeper, vatLoader, policyOptions) { async function resetWorker(vatID) { await evict(vatID); const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - vatKeeper.removeSnapshotAndTranscript(); + vatKeeper.removeSnapshotAndResetTranscript(); } /** diff --git a/packages/SwingSet/test/device-plugin/test-device.js b/packages/SwingSet/test/device-plugin/test-device.js index 307587e3de6..7b38950c373 100644 --- a/packages/SwingSet/test/device-plugin/test-device.js +++ b/packages/SwingSet/test/device-plugin/test-device.js @@ -90,6 +90,12 @@ test.serial('plugin first time', async t => { ]); }); +// NOTE: the following test CANNOT be run standalone. It requires execution of +// the prior test to establish its necessary starting state. This is a bad +// practice and should be fixed. It's not bad enough to warrant fixing right +// now, but worth flagging with this comment as a help to anyone else who gets +// tripped up by it. + test.serial('plugin after restart', async t => { const { bridge, cycle, dump, plugin, queueThunkForKernel } = await setupVatController(t); diff --git a/packages/SwingSet/test/vat-admin/terminate/test-terminate.js b/packages/SwingSet/test/vat-admin/terminate/test-terminate.js index 2a98d2f5669..1bc8c73572c 100644 --- a/packages/SwingSet/test/vat-admin/terminate/test-terminate.js +++ b/packages/SwingSet/test/vat-admin/terminate/test-terminate.js @@ -443,7 +443,7 @@ test.serial('dead vat state removed', async t => { const configPath = new URL('swingset-die-cleanly.json', import.meta.url) .pathname; const config = await loadSwingsetConfigFile(configPath); - const kernelStorage = initSwingStore().kernelStorage; + const { kernelStorage, debug } = initSwingStore(); const controller = await buildVatController(config, [], { kernelStorage, @@ -456,16 +456,22 @@ test.serial('dead vat state removed', async t => { controller.kpResolution(controller.bootstrapResult), kser('bootstrap done'), ); - const kvStore = kernelStorage.kvStore; + const { kvStore } = kernelStorage; t.is(kvStore.get('vat.dynamicIDs'), '["v6"]'); t.is(kvStore.get('ko26.owner'), 'v6'); t.is(Array.from(enumeratePrefixedKeys(kvStore, 'v6.')).length > 30, true); + const beforeDump = debug.dump(true); + t.truthy(beforeDump.transcripts.v6); + t.truthy(beforeDump.snapshots.v6); controller.queueToVatRoot('bootstrap', 'phase2', []); await controller.run(); t.is(kvStore.get('vat.dynamicIDs'), '[]'); t.is(kvStore.get('ko26.owner'), undefined); t.is(Array.from(enumeratePrefixedKeys(kvStore, 'v6.')).length, 0); + const afterDump = debug.dump(true); + t.falsy(afterDump.transcripts.v6); + t.falsy(afterDump.snapshots.v6); }); test.serial('terminate with presence', async t => { diff --git a/packages/swing-store/src/snapStore.js b/packages/swing-store/src/snapStore.js index d0bea79d2a6..3de8a04e566 100644 --- a/packages/swing-store/src/snapStore.js +++ b/packages/swing-store/src/snapStore.js @@ -37,7 +37,7 @@ import { fsStreamReady } from '@agoric/internal/src/fs-stream.js'; * }} SnapStore * * @typedef {{ - * exportSnapshot: (vatID: string, endPos: number) => AsyncIterable, + * exportSnapshot: (name: string, includeHistorical: boolean) => AsyncIterable, * importSnapshot: (artifactName: string, exporter: SwingStoreExporter, artifactMetadata: Map) => void, * getExportRecords: (includeHistorical: boolean) => Iterable<[key: string, value: string]>, * getArtifactNames: (includeHistorical: boolean) => AsyncIterable, @@ -84,7 +84,7 @@ const noPath = /** @type {import('fs').PathLike} */ ( * tmpName: typeof import('tmp').tmpName, * unlink: typeof import('fs').promises.unlink, * }} io - * @param {(key: string, value: string) => void} noteExport + * @param {(key: string, value: string | undefined) => void} noteExport * @param {object} [options] * @param {boolean | undefined} [options.keepSnapshots] * @returns {SnapStore & SnapStoreInternal & SnapStoreDebug} @@ -157,10 +157,22 @@ export function makeSnapStore( return `snapshot.${rec.vatID}.current`; } - function snapshotRec(vatID, endPos, hash) { - return { vatID, endPos, hash }; + function snapshotRec(vatID, endPos, hash, inUse) { + return { vatID, endPos, hash, inUse }; } + const sqlGetPriorSnapshotInfo = db.prepare(` + SELECT endPos, hash + FROM snapshots + WHERE vatID = ? AND inUse = 1 + `); + + const sqlClearLastSnapshot = db.prepare(` + UPDATE snapshots + SET inUse = 0, compressedSnapshot = null + WHERE inUse = 1 AND vatID = ? + `); + const sqlStopUsingLastSnapshot = db.prepare(` UPDATE snapshots SET inUse = 0 @@ -169,8 +181,8 @@ export function makeSnapStore( const sqlSaveSnapshot = db.prepare(` INSERT OR REPLACE INTO snapshots - (vatID, endPos, inUse, hash, uncompressedSize, compressedSize, compressedSnapshot) - VALUES (?, ?, 1, ?, ?, ?, ?) + (vatID, endPos, hash, uncompressedSize, compressedSize, compressedSnapshot, inUse) + VALUES (?, ?, ?, ?, ?, ?, ?) `); /** @@ -217,9 +229,15 @@ export function makeSnapStore( await finished(snapReader); const h = hashStream.digest('hex'); - sqlStopUsingLastSnapshot.run(vatID); - if (!keepSnapshots) { - deleteAllUnusedSnapshots(); + const oldInfo = sqlGetPriorSnapshotInfo.get(vatID); + if (oldInfo) { + const rec = snapshotRec(vatID, oldInfo.endPos, oldInfo.hash, 0); + noteExport(snapshotMetadataKey(rec), JSON.stringify(rec)); + if (keepSnapshots) { + sqlStopUsingLastSnapshot.run(vatID); + } else { + sqlClearLastSnapshot.run(vatID); + } } compressedSize = compressedSnapshot.length; sqlSaveSnapshot.run( @@ -229,8 +247,9 @@ export function makeSnapStore( uncompressedSize, compressedSize, compressedSnapshot, + 1, ); - const rec = snapshotRec(vatID, endPos, h); + const rec = snapshotRec(vatID, endPos, h, 1); const exportKey = snapshotMetadataKey(rec); noteExport(exportKey, JSON.stringify(rec)); noteExport( @@ -257,24 +276,43 @@ export function makeSnapStore( } const sqlGetSnapshot = db.prepare(` - SELECT compressedSnapshot + SELECT compressedSnapshot, inUse FROM snapshots WHERE vatID = ? AND endPos = ? `); - sqlGetSnapshot.pluck(true); /** - * @param {string} vatID - * @param {number} endPos + * Read a snapshot and return it as a stream of data suitable for export to + * another store. + * + * Snapshot artifact names should be strings of the form: + * `snapshot.${vatID}.${startPos}` + * + * @param {string} name + * @param {boolean} includeHistorical * @returns {AsyncIterable} - * @yields {Uint8Array} */ - async function* exportSnapshot(vatID, endPos) { - const compressedSnapshot = sqlGetSnapshot.get(vatID, endPos); - const gzReader = Readable.from(compressedSnapshot); - const unzipper = createGunzip(); - const snapshotReader = gzReader.pipe(unzipper); - yield* snapshotReader; + function exportSnapshot(name, includeHistorical) { + typeof name === 'string' || Fail`artifact name must be a string`; + const parts = name.split('.'); + const [type, vatID, pos] = parts; + // prettier-ignore + (parts.length === 3 && type === 'snapshot') || + Fail`expected artifact name of the form 'snapshot.{vatID}.{endPos}', saw ${q(name)}`; + const endPos = Number(pos); + const snapshotInfo = sqlGetSnapshot.get(vatID, endPos); + snapshotInfo || Fail`snapshot ${q(name)} not available`; + const { inUse, compressedSnapshot } = snapshotInfo; + compressedSnapshot || Fail`artifact ${q(name)} is not available`; + inUse || includeHistorical || Fail`artifact ${q(name)} is not available`; + // weird construct here is because we need to be able to throw before the generator starts + async function* exporter() { + const gzReader = Readable.from(compressedSnapshot); + const unzipper = createGunzip(); + const snapshotReader = gzReader.pipe(unzipper); + yield* snapshotReader; + } + return exporter(); } const sqlLoadSnapshot = db.prepare(` @@ -341,12 +379,23 @@ export function makeSnapStore( WHERE vatID = ? `); + const sqlGetSnapshotList = db.prepare(` + SELECT endPos + FROM snapshots + WHERE vatID = ? + `); + /** * Delete all snapshots for a given vat (for use when, e.g., a vat is terminated) * * @param {string} vatID */ function deleteVatSnapshots(vatID) { + for (const rec of sqlGetSnapshotList.iterate(vatID)) { + const exportRec = snapshotRec(vatID, rec.endPos, undefined); + noteExport(snapshotMetadataKey(exportRec), undefined); + } + noteExport(currentSnapshotMetadataKey({ vatID }), undefined); sqlDeleteVatSnapshots.run(vatID); } @@ -411,27 +460,42 @@ export function makeSnapStore( } const sqlGetSnapshotMetadata = db.prepare(` - SELECT vatID, endPos, hash, uncompressedSize, compressedSize + SELECT vatID, endPos, hash, uncompressedSize, compressedSize, inUse FROM snapshots WHERE inUse = ? ORDER BY vatID, endPos `); /** - * @param {boolean} includeHistorical + * Obtain artifact metadata records for spanshots contained in this store. + * + * @param {boolean} includeHistorical If true, include all metadata that is + * present in the store regardless of its currency; if false, only include + * the metadata that is part of the swingset's active operational state. + * + * Note: in the currently anticipated operational mode, this flag should + * always be set to `true`, because *all* snapshot metadata is, for now, + * considered part of the consensus set. This metadata is being retained for + * diagnostic purposes and as a hedge against possible future need. While + * such a need seems highly unlikely, the future is uncertain and it will be + * easier to purge this data later than to recover it if it is lost. However, + * the flag itself is present in case future operational policy allows for + * pruning historical metadata, for example after further analysis and + * practical experience tells us that it will not be needed. + * * @yields {[key: string, value: string]} * @returns {Iterable<[key: string, value: string]>} */ - function* getExportRecords(includeHistorical) { + function* getExportRecords(includeHistorical = true) { for (const rec of sqlGetSnapshotMetadata.iterate(1)) { - const exportRec = snapshotRec(rec.vatID, rec.endPos, rec.hash); + const exportRec = snapshotRec(rec.vatID, rec.endPos, rec.hash, 1); const exportKey = snapshotMetadataKey(rec); yield [exportKey, JSON.stringify(exportRec)]; yield [currentSnapshotMetadataKey(rec), snapshotArtifactName(rec)]; } if (includeHistorical) { for (const rec of sqlGetSnapshotMetadata.iterate(0)) { - const exportRec = snapshotRec(rec.vatID, rec.endPos, rec.hash); + const exportRec = snapshotRec(rec.vatID, rec.endPos, rec.hash, 0); yield [snapshotMetadataKey(rec), JSON.stringify(exportRec)]; } } @@ -489,18 +553,19 @@ export function makeSnapStore( size, compressedArtifact.length, compressedArtifact, + info.inUse, ); } const sqlDumpCurrentSnapshots = db.prepare(` - SELECT vatID, endPos, hash, compressedSnapshot + SELECT vatID, endPos, hash, compressedSnapshot, inUse FROM snapshots WHERE inUse = 1 ORDER BY vatID, endPos `); const sqlDumpAllSnapshots = db.prepare(` - SELECT vatID, endPos, hash, compressedSnapshot + SELECT vatID, endPos, hash, compressedSnapshot, inUse FROM snapshots ORDER BY vatID, endPos `); @@ -516,8 +581,11 @@ export function makeSnapStore( : sqlDumpCurrentSnapshots; const dump = {}; for (const row of sql.iterate()) { - const { vatID, endPos, hash, compressedSnapshot } = row; - dump[vatID] = { endPos, hash, compressedSnapshot }; + const { vatID, endPos, hash, compressedSnapshot, inUse } = row; + if (!dump[vatID]) { + dump[vatID] = []; + } + dump[vatID].push({ endPos, hash, compressedSnapshot, inUse }); } return dump; } diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index 0b3d27618bb..5fb25c2de26 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -78,10 +78,7 @@ function getKeyType(key) { * setExportCallback: (cb: (updates: KVPair[]) => void) => void, // Set a callback invoked by swingStore when new serializable data is available for export * }} SwingStoreHostStorage */ -/* - * getExporter(): SwingStoreExporter, // Creates an exporter of the swingStore's content as of - * // the most recent commit point - */ + /** * @typedef {{ * kvEntries: {}, @@ -137,7 +134,7 @@ function getKeyType(key) { * - transcript.${vatID}.${startPos} = ${{ vatID, startPos, endPos, hash }} * - transcript.${vatID}.current = ${{ vatID, startPos, endPos, hash }} * - * @property {(includeHistorical: boolean) => AsyncIterable} getArtifactNames + * @property {() => AsyncIterable} getArtifactNames * * Get a list of name of artifacts available from the swingStore. A name returned * by this method guarantees that a call to `getArtifact` on the same exporter @@ -161,10 +158,17 @@ function getKeyType(key) { /** * @param {string} dirPath + * @param {string} exportMode * @returns {SwingStoreExporter} */ -export function makeSwingStoreExporter(dirPath) { +export function makeSwingStoreExporter(dirPath, exportMode = 'current') { typeof dirPath === 'string' || Fail`dirPath must be a string`; + exportMode === 'current' || + exportMode === 'archival' || + exportMode === 'debug' || + Fail`invalid exportMode ${q(exportMode)}`; + const exportHistoricalSnapshots = exportMode === 'debug'; + const exportHistoricalTranscripts = exportMode !== 'current'; const filePath = path.join(dirPath, 'swingstore.sqlite'); const db = sqlite3(filePath); @@ -202,13 +206,12 @@ export function makeSwingStoreExporter(dirPath) { } /** - * @param {boolean} includeHistorical * @returns {AsyncIterable} * @yields {string} */ - async function* getArtifactNames(includeHistorical) { - yield* snapStore.getArtifactNames(includeHistorical); - yield* transcriptStore.getArtifactNames(includeHistorical); + async function* getArtifactNames() { + yield* snapStore.getArtifactNames(exportHistoricalSnapshots); + yield* transcriptStore.getArtifactNames(exportHistoricalTranscripts); } /** @@ -217,21 +220,12 @@ export function makeSwingStoreExporter(dirPath) { */ function getArtifact(name) { typeof name === 'string' || Fail`artifact name must be a string`; - const parts = name.split('.'); - const [type, vatID, pos] = parts; + const [type] = name.split('.', 1); if (type === 'snapshot') { - // `snapshot.${vatID}.${endPos}`; - // prettier-ignore - parts.length === 3 || - Fail`expected artifact name of the form 'snapshot.{vatID}.{endPos}', saw ${q(name)}`; - return snapStore.exportSnapshot(vatID, Number(pos)); + return snapStore.exportSnapshot(name, exportHistoricalSnapshots); } else if (type === 'transcript') { - // `transcript.${vatID}.${startPos}.${endPos}`; - // prettier-ignore - parts.length === 4 || - Fail`expected artifact name of the form 'transcript.{vatID}.{startPos}.{endPos}', saw ${q(name)}`; - return transcriptStore.exportSpan(vatID, Number(pos)); + return transcriptStore.exportSpan(name, exportHistoricalTranscripts); } else { assert.fail(`invalid artifact type ${q(type)}`); } @@ -356,7 +350,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) { filePath = ':memory:'; } - const { traceFile, keepSnapshots } = options; + const { traceFile, keepSnapshots, keepTranscripts } = options; let traceOutput = traceFile ? fs.createWriteStream(path.resolve(traceFile), { @@ -647,6 +641,9 @@ function makeSwingStore(dirPath, forceReset, options = {}) { db, ensureTxn, noteExport, + { + keepTranscripts, + }, ); const { dumpSnapshots, ...snapStore } = makeSnapStore( db, @@ -812,6 +809,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) { getCurrentSpanBounds: transcriptStore.getCurrentSpanBounds, addItem: transcriptStore.addItem, readSpan: transcriptStore.readSpan, + deleteVatTranscripts: transcriptStore.deleteVatTranscripts, }; const snapStorePublic = { @@ -1019,7 +1017,7 @@ export async function importSwingStore(exporter, dirPath = null, options = {}) { // If we're also importing historical artifacts, have the exporter enumerate // the complete set of artifacts it has and fetch all of them except for the // ones we've already fetched. - for await (const artifactName of exporter.getArtifactNames(true)) { + for await (const artifactName of exporter.getArtifactNames()) { if (fetchedArtifacts.has(artifactName)) { continue; } diff --git a/packages/swing-store/src/transcriptStore.js b/packages/swing-store/src/transcriptStore.js index 68527e381e5..4825412b1d9 100644 --- a/packages/swing-store/src/transcriptStore.js +++ b/packages/swing-store/src/transcriptStore.js @@ -12,12 +12,13 @@ import { createSHA256 } from './hasher.js'; * initTranscript: (vatID: string) => void, * rolloverSpan: (vatID: string) => void, * getCurrentSpanBounds: (vatID: string) => { startPos: number, endPos: number, hash: string }, + * deleteVatTranscripts: (vatID: string) => void, * addItem: (vatID: string, item: string) => void, * readSpan: (vatID: string, startPos?: number) => Iterable, * }} TranscriptStore * * @typedef {{ - * exportSpan: (vatID: string, startPos: number) => AsyncIterable + * exportSpan: (name: string, includeHistorical: boolean) => AsyncIterable * importSpan: (artifactName: string, exporter: SwingStoreExporter, artifactMetadata: Map) => Promise, * getExportRecords: (includeHistorical: boolean) => Iterable<[key: string, value: string]>, * getArtifactNames: (includeHistorical: boolean) => AsyncIterable, @@ -46,10 +47,17 @@ function insistTranscriptPosition(position) { /** * @param {*} db * @param {() => void} ensureTxn - * @param {(key: string, value: string) => void} noteExport + * @param {(key: string, value: string | undefined ) => void} noteExport + * @param {object} [options] + * @param {boolean | undefined} [options.keepTranscripts] * @returns { TranscriptStore & TranscriptStoreInternal & TranscriptStoreDebug } */ -export function makeTranscriptStore(db, ensureTxn, noteExport) { +export function makeTranscriptStore( + db, + ensureTxn, + noteExport, + { keepTranscripts = true } = {}, +) { db.exec(` CREATE TABLE IF NOT EXISTS transcriptItems ( vatID TEXT, @@ -83,6 +91,10 @@ export function makeTranscriptStore(db, ensureTxn, noteExport) { PRIMARY KEY (vatID, startPos) ) `); + db.exec(` + CREATE INDEX IF NOT EXISTS currentTranscriptIndex + ON transcriptSpans (vatID, isCurrent) + `); const sqlDumpItemsQuery = db.prepare(` SELECT vatID, position, item @@ -200,6 +212,11 @@ export function makeTranscriptStore(db, ensureTxn, noteExport) { WHERE isCurrent = 1 AND vatID = ? `); + const sqlDeleteOldItems = db.prepare(` + DELETE FROM transcriptItems + WHERE vatID = ? AND position < ? + `); + /** * End the current transcript span for a vat and start a new one. * @@ -215,6 +232,41 @@ export function makeTranscriptStore(db, ensureTxn, noteExport) { sqlWriteSpan.run(vatID, endPos, endPos, initialHash, 1); const newRec = spanRec(vatID, endPos, endPos, initialHash, 1); noteExport(currentSpanMetadataKey(newRec), JSON.stringify(newRec)); + if (!keepTranscripts) { + sqlDeleteOldItems.run(vatID, endPos); + } + } + + const sqlDeleteVatSpans = db.prepare(` + DELETE FROM transcriptSpans + WHERE vatID = ? + `); + + const sqlDeleteVatItems = db.prepare(` + DELETE FROM transcriptItems + WHERE vatID = ? + `); + + const sqlGetVatSpans = db.prepare(` + SELECT startPos + FROM transcriptSpans + WHERE vatID = ? + `); + sqlGetVatSpans.pluck(true); + + /** + * Delete all transcript data for a given vat (for use when, e.g., a vat is terminated) + * + * @param {string} vatID + */ + function deleteVatTranscripts(vatID) { + for (const startPos of sqlGetVatSpans.iterate(vatID)) { + const exportRec = spanRec(vatID, startPos); + noteExport(historicSpanMetadataKey(exportRec), undefined); + } + noteExport(currentSpanMetadataKey({ vatID }), undefined); + sqlDeleteVatItems.run(vatID); + sqlDeleteVatSpans.run(vatID); } const sqlGetAllSpanMetadata = db.prepare(` @@ -233,16 +285,31 @@ export function makeTranscriptStore(db, ensureTxn, noteExport) { /** * Obtain artifact metadata records for spans contained in this store. * - * @param {boolean} includeHistorical If true, include all spans that are - * present in the store regardless of their currency; if false, only include - * the current span for each vat. + * @param {boolean} includeHistorical If true, include all metadata that is + * present in the store regardless of its currency; if false, only include + * the metadata that is part of the swingset's active operational state. + * + * Note: in the currently anticipated operational mode, this flag should + * always be set to `true`, because *all* transcript span metadata is, for + * now, considered part of the consensus set. This metadata is being retained + * as a hedge against possible future need, wherein we find it necessary to + * replay a vat's entire history from t0 and therefor need to be able to + * validate historical transcript artifacts that were recovered from external + * archives rather than retained directly. While such a need seems highly + * unlikely, it hypothetically could be forced by some necessary vat upgrade + * that implicates path-dependent ephemeral state despite our best efforts to + * avoid having any such state. However, the flag itself is present in case + * future operational policy allows for pruning historical transcript span + * metadata, for example because we've determined that such full-history + * replay will never be required or because such replay would be prohibitively + * expensive regardless of need and therefor other repair strategies employed. * * @yields {[key: string, value: string]} * @returns {Iterable<[key: string, value: string]>} An iterator over pairs of * [historicSpanMetadataKey, rec], where `rec` is a JSON-encoded metadata record for the * span named by `historicSpanMetadataKey`. */ - function* getExportRecords(includeHistorical) { + function* getExportRecords(includeHistorical = true) { const sql = includeHistorical ? sqlGetAllSpanMetadata : sqlGetCurrentSpanMetadata; @@ -325,17 +392,39 @@ export function makeTranscriptStore(db, ensureTxn, noteExport) { return reader(); } + const sqlGetSpanIsCurrent = db.prepare(` + SELECT isCurrent + FROM transcriptSpans + WHERE vatID = ? AND startPos = ? + `); + sqlGetSpanIsCurrent.pluck(true); + /** * Read a transcript span and return it as a stream of data suitable for - * export to another store. Items are terminated by newlines. + * export to another store. Transcript items are terminated by newlines. * - * @param {string} vatID The vat whose transcript is being read - * @param {number} [startPos] A start position identifying the span to be read + * Transcript span artifact names should be strings of the form: + * `transcript.${vatID}.${startPos}.${endPos}` + * + * @param {string} name The name of the transcript artifact to be read + * @param {boolean} includeHistorical If true, allow non-current spans to be fetched * * @returns {AsyncIterable} * @yields {Uint8Array} */ - async function* exportSpan(vatID, startPos) { + async function* exportSpan(name, includeHistorical) { + typeof name === 'string' || Fail`artifact name must be a string`; + const parts = name.split('.'); + const [type, vatID, pos] = parts; + // prettier-ignore + (parts.length === 4 && type === 'transcript') || + Fail`expected artifact name of the form 'transcript.{vatID}.{startPos}.{endPos}', saw ${q(name)}`; + const isCurrent = sqlGetSpanIsCurrent.get(vatID, pos); + isCurrent !== undefined || Fail`transcript span ${q(name)} not available`; + isCurrent || + includeHistorical || + Fail`transcript span ${q(name)} not available`; + const startPos = Number(pos); for (const entry of readSpan(vatID, startPos)) { yield Buffer.from(`${entry}\n`); } @@ -406,6 +495,7 @@ export function makeTranscriptStore(db, ensureTxn, noteExport) { hash = computeItemHash(hash, item); pos += 1; } + pos === endPos || Fail`artifact ${name} is not available`; info.hash === hash || Fail`artifact ${name} hash is ${q(hash)}, metadata says ${q(info.hash)}`; sqlWriteSpan.run( @@ -423,6 +513,7 @@ export function makeTranscriptStore(db, ensureTxn, noteExport) { getCurrentSpanBounds, addItem, readSpan, + deleteVatTranscripts, exportSpan, importSpan, diff --git a/packages/swing-store/test/test-exportImport.js b/packages/swing-store/test/test-exportImport.js new file mode 100644 index 00000000000..952a1c3a0a1 --- /dev/null +++ b/packages/swing-store/test/test-exportImport.js @@ -0,0 +1,442 @@ +// @ts-check + +import '@endo/init/debug.js'; +import fs from 'fs'; + +// eslint-disable-next-line import/no-extraneous-dependencies +import test from 'ava'; +// eslint-disable-next-line import/no-extraneous-dependencies +import tmp from 'tmp'; + +import { + initSwingStore, + makeSwingStoreExporter, + importSwingStore, +} from '../src/swingStore.js'; + +function makeExportLog() { + const exportLog = []; + const shadowStore = new Map(); + return { + callback(updates) { + exportLog.push(updates); + for (const [key, value] of updates) { + if (value == null) { + shadowStore.delete(key); + } else { + shadowStore.set(key, value); + } + } + }, + getLog() { + return exportLog; + }, + entries() { + return shadowStore.entries(); + }, + }; +} + +/** + * @param {string} [prefix] + * @returns {Promise<[string, () => void]>} + */ +const tmpDir = prefix => + new Promise((resolve, reject) => { + tmp.dir({ unsafeCleanup: true, prefix }, (err, name, removeCallback) => { + if (err) { + reject(err); + } else { + resolve([name, removeCallback]); + } + }); + }); + +function actLikeAVatRunningACrank(vat, ks, crank, doFail) { + const { kvStore, transcriptStore } = ks; + const { vatID } = vat; + ks.startCrank(); + if (doFail) { + ks.establishCrankSavepoint('a'); + } + kvStore.set('kval', `set in ${crank}`); + kvStore.set(`${vatID}.vval`, `stuff in ${crank}`); + kvStore.set(`${vatID}.monotonic.${crank}`, 'more and more'); + if (crank % 3 === 0) { + kvStore.delete('brigadoon'); + } else { + kvStore.set('brigadoon', `here during ${crank}`); + } + transcriptStore.addItem(vatID, `stuff done during crank #${crank}`); + if (doFail) { + ks.rollbackCrank('a'); + } else { + vat.endPos += 1; + } + ks.endCrank(); +} + +async function fakeAVatSnapshot(vat, ks) { + await ks.snapStore.saveSnapshot(vat.vatID, vat.endPos, async filePath => { + fs.writeFileSync( + filePath, + `snapshot of vat ${vat.vatID} as of ${vat.endPos}`, + ); + }); + ks.transcriptStore.rolloverSpan(vat.vatID); +} + +const compareElems = (a, b) => a[0].localeCompare(b[0]); + +test('crank abort leaves no debris in export log', async t => { + const exportLog = makeExportLog(); + const [dbDir, cleanup] = await tmpDir('testdb'); + t.teardown(cleanup); + + const ssOut = initSwingStore(dbDir, { + exportCallback: exportLog.callback, + }); + const { kernelStorage } = ssOut; + + const vat = { vatID: 'vatA', endPos: 0 }; + kernelStorage.transcriptStore.initTranscript(vat.vatID); + + // Run 4 "blocks", each consisting of 4 "cranks", accumulating stuff, + // aborting every third crank. + let crankNum = 0; + for (let block = 0; block < 4; block += 1) { + for (let crank = 0; crank < 4; crank += 1) { + crankNum += 1; + actLikeAVatRunningACrank( + vat, + kernelStorage, + crankNum, + crankNum % 3 === 0, + ); + } + ssOut.hostStorage.commit(); + } + + const exporter = makeSwingStoreExporter(dbDir, 'current'); + + const exportData = []; + for await (const elem of exporter.getExportData()) { + exportData.push(elem); + } + exportData.sort(compareElems); + + const feedData = []; + for (const elem of exportLog.entries()) { + feedData.push(elem); + } + feedData.sort(compareElems); + + t.deepEqual(exportData, feedData); + // Commented data entries would have been produced by the aborted cranks + t.deepEqual(exportData, [ + ['kv.brigadoon', 'here during 16'], + ['kv.kval', 'set in 16'], + ['kv.vatA.monotonic.1', 'more and more'], + ['kv.vatA.monotonic.10', 'more and more'], + ['kv.vatA.monotonic.11', 'more and more'], + // ['kv.vatA.monotonic.12', 'more and more'], + ['kv.vatA.monotonic.13', 'more and more'], + ['kv.vatA.monotonic.14', 'more and more'], + // ['kv.vatA.monotonic.15', 'more and more'], + ['kv.vatA.monotonic.16', 'more and more'], + ['kv.vatA.monotonic.2', 'more and more'], + // ['kv.vatA.monotonic.3', 'more and more'], + ['kv.vatA.monotonic.4', 'more and more'], + ['kv.vatA.monotonic.5', 'more and more'], + // ['kv.vatA.monotonic.6', 'more and more'], + ['kv.vatA.monotonic.7', 'more and more'], + ['kv.vatA.monotonic.8', 'more and more'], + // ['kv.vatA.monotonic.9', 'more and more'], + ['kv.vatA.vval', 'stuff in 16'], + [ + 'transcript.vatA.current', + // '{"vatID":"vatA","startPos":0,"endPos":16,"hash":"83e7ed8d3ee339a8b0989512973396d3e9db4b4c3d76570862d99e3cdebaf8c6","isCurrent":1}', + '{"vatID":"vatA","startPos":0,"endPos":11,"hash":"ff988824e0fb02bfd0a5ecf466513fd4ef2ac6e488ab9070e640683faa8ddb11","isCurrent":1}', + ], + ]); +}); + +async function testExportImport( + t, + runMode, + exportMode, + importMode, + failureMode, + expectedArtifactNames, +) { + const exportLog = makeExportLog(); + const [dbDir, cleanup] = await tmpDir('testdb'); + t.teardown(cleanup); + + const keepTranscripts = runMode !== 'current'; + const keepSnapshots = runMode === 'debug'; + const ssOut = initSwingStore(dbDir, { + exportCallback: exportLog.callback, + keepSnapshots, + keepTranscripts, + }); + const { kernelStorage, debug } = ssOut; + + const vats = [ + { vatID: 'vatA', endPos: 0 }, + { vatID: 'vatB', endPos: 0 }, + ]; + for (const vat of vats) { + kernelStorage.transcriptStore.initTranscript(vat.vatID); + } + + // Run 4 "blocks", each consisting of 4 "cranks", across 2 "vats" + // Snapshot 'vatA' after the first and third blocks and 'vatB' after the second + // This will leave 2 current snapshots and 1 historical snapshot + let crankNum = 0; + for (let block = 0; block < 4; block += 1) { + for (let crank = 0; crank < 4; crank += 1) { + crankNum += 1; + const vat = vats[crankNum % vats.length]; + actLikeAVatRunningACrank(vat, kernelStorage, crankNum); + } + if (block < 3) { + // eslint-disable-next-line no-await-in-loop + await fakeAVatSnapshot(vats[block % 2], kernelStorage); + } + ssOut.hostStorage.commit(); + } + + const exporter = makeSwingStoreExporter(dbDir, exportMode); + + const exportData = []; + for await (const elem of exporter.getExportData()) { + exportData.push(elem); + } + exportData.sort(compareElems); + + const feedData = []; + for (const elem of exportLog.entries()) { + feedData.push(elem); + } + feedData.sort(compareElems); + + t.deepEqual(exportData, feedData); + t.deepEqual(exportData, [ + ['kv.brigadoon', 'here during 16'], + ['kv.kval', 'set in 16'], + ['kv.vatA.monotonic.10', 'more and more'], + ['kv.vatA.monotonic.12', 'more and more'], + ['kv.vatA.monotonic.14', 'more and more'], + ['kv.vatA.monotonic.16', 'more and more'], + ['kv.vatA.monotonic.2', 'more and more'], + ['kv.vatA.monotonic.4', 'more and more'], + ['kv.vatA.monotonic.6', 'more and more'], + ['kv.vatA.monotonic.8', 'more and more'], + ['kv.vatA.vval', 'stuff in 16'], + ['kv.vatB.monotonic.1', 'more and more'], + ['kv.vatB.monotonic.11', 'more and more'], + ['kv.vatB.monotonic.13', 'more and more'], + ['kv.vatB.monotonic.15', 'more and more'], + ['kv.vatB.monotonic.3', 'more and more'], + ['kv.vatB.monotonic.5', 'more and more'], + ['kv.vatB.monotonic.7', 'more and more'], + ['kv.vatB.monotonic.9', 'more and more'], + ['kv.vatB.vval', 'stuff in 15'], + [ + 'snapshot.vatA.2', + '{"vatID":"vatA","endPos":2,"hash":"6c7e452ee3eaec849c93234d933af4300012e4ff161c328d3c088ec3deef76a6","inUse":0}', + ], + [ + 'snapshot.vatA.6', + '{"vatID":"vatA","endPos":6,"hash":"36afc9e2717c395759e308c4a877d491f967e9768d73520bde758ff4fac5d8b9","inUse":1}', + ], + ['snapshot.vatA.current', 'snapshot.vatA.6'], + [ + 'snapshot.vatB.4', + '{"vatID":"vatB","endPos":4,"hash":"afd477014db678fbc1aa58beab50f444deb653b8cc8e8583214a363fd12ed57a","inUse":1}', + ], + ['snapshot.vatB.current', 'snapshot.vatB.4'], + [ + 'transcript.vatA.0', + '{"vatID":"vatA","startPos":0,"endPos":2,"hash":"ea8ac1a751712ad66e4a9182adc65afe9bb0f4cd0ee0b828c895c63fbd2e3157","isCurrent":0}', + ], + [ + 'transcript.vatA.2', + '{"vatID":"vatA","startPos":2,"endPos":6,"hash":"88f299ca67b8acdf6023a83bb8e899af5adcf3271c7a1a2a495dcd6f1fbaac9f","isCurrent":0}', + ], + [ + 'transcript.vatA.current', + '{"vatID":"vatA","startPos":6,"endPos":8,"hash":"fe5d692b24a32d53bf617ba9ed3391b60c36a402c70a07a6aa984fc316e4efcc","isCurrent":1}', + ], + [ + 'transcript.vatB.0', + '{"vatID":"vatB","startPos":0,"endPos":4,"hash":"41dbf60cdec066106c7030517cb9f9f34a50fe2259705cf5fdbdd0b39ae12e46","isCurrent":0}', + ], + [ + 'transcript.vatB.current', + '{"vatID":"vatB","startPos":4,"endPos":8,"hash":"34fa09207bfb7af5fc3e65acb07f13b60834d0fbd2c6b9708f794c4397bd865d","isCurrent":1}', + ], + ]); + + const artifactNames = []; + for await (const name of exporter.getArtifactNames()) { + artifactNames.push(name); + } + t.deepEqual(artifactNames, expectedArtifactNames); + + const includeHistorical = importMode !== 'current'; + + const beforeDump = debug.dump(keepSnapshots); + let ssIn; + try { + ssIn = await importSwingStore(exporter, null, { + includeHistorical, + }); + } catch (e) { + if (failureMode === 'transcript') { + t.is(e.message, 'artifact "transcript.vatA.0.2" is not available'); + return; + } else if (failureMode === 'snapshot') { + t.is(e.message, 'artifact "snapshot.vatA.2" is not available'); + return; + } + throw e; + } + t.is(failureMode, 'none'); + ssIn.hostStorage.commit(); + const dumpsShouldMatch = + runMode !== 'debug' || (exportMode === 'debug' && importMode !== 'current'); + if (dumpsShouldMatch) { + const afterDump = ssIn.debug.dump(keepSnapshots); + t.deepEqual(beforeDump, afterDump); + } + + exporter.close(); +} + +const expectedCurrentArtifacts = [ + 'snapshot.vatA.6', + 'snapshot.vatB.4', + 'transcript.vatA.6.8', + 'transcript.vatB.4.8', +]; + +const expectedArchivalArtifacts = [ + 'snapshot.vatA.6', + 'snapshot.vatB.4', + 'transcript.vatA.0.2', + 'transcript.vatA.2.6', + 'transcript.vatA.6.8', + 'transcript.vatB.0.4', + 'transcript.vatB.4.8', +]; + +const expectedDebugArtifacts = [ + 'snapshot.vatA.6', + 'snapshot.vatB.4', + 'snapshot.vatA.2', + 'transcript.vatA.0.2', + 'transcript.vatA.2.6', + 'transcript.vatA.6.8', + 'transcript.vatB.0.4', + 'transcript.vatB.4.8', +]; + +const C = 'current'; +const A = 'archival'; +const D = 'debug'; + +test('export and import data for state sync - current->current->current', async t => { + await testExportImport(t, C, C, C, 'none', expectedCurrentArtifacts); +}); +test('export and import data for state sync - current->current->archival', async t => { + await testExportImport(t, C, C, A, 'none', expectedCurrentArtifacts); +}); +test('export and import data for state sync - current->current->debug', async t => { + await testExportImport(t, C, C, D, 'none', expectedCurrentArtifacts); +}); + +test('export and import data for state sync - current->archival->current', async t => { + await testExportImport(t, C, A, C, 'none', expectedArchivalArtifacts); +}); +test('export and import data for state sync - current->archival->archival', async t => { + await testExportImport(t, C, A, A, 'transcript', expectedArchivalArtifacts); +}); +test('export and import data for state sync - current->archival->debug', async t => { + await testExportImport(t, C, A, D, 'transcript', expectedArchivalArtifacts); +}); + +test('export and import data for state sync - current->debug->current', async t => { + await testExportImport(t, C, D, C, 'none', expectedDebugArtifacts); +}); +test('export and import data for state sync - current->debug->archival', async t => { + await testExportImport(t, C, D, A, 'snapshot', expectedDebugArtifacts); +}); +test('export and import data for state sync - current->debug->debug', async t => { + await testExportImport(t, C, D, D, 'snapshot', expectedDebugArtifacts); +}); + +// ------------------------------------------------------------ + +test('export and import data for state sync - archival->current->current', async t => { + await testExportImport(t, A, C, C, 'none', expectedCurrentArtifacts); +}); +test('export and import data for state sync - archival->current->archival', async t => { + await testExportImport(t, A, C, A, 'none', expectedCurrentArtifacts); +}); +test('export and import data for state sync - archival->current->debug', async t => { + await testExportImport(t, A, C, D, 'none', expectedCurrentArtifacts); +}); + +test('export and import data for state sync - archival->archival->current', async t => { + await testExportImport(t, A, A, C, 'none', expectedArchivalArtifacts); +}); +test('export and import data for state sync - archival->archival->archival', async t => { + await testExportImport(t, A, A, A, 'none', expectedArchivalArtifacts); +}); +test('export and import data for state sync - archival->archival->debug', async t => { + await testExportImport(t, A, A, D, 'none', expectedArchivalArtifacts); +}); + +test('export and import data for state sync - archival->debug->current', async t => { + await testExportImport(t, A, D, C, 'none', expectedDebugArtifacts); +}); +test('export and import data for state sync - archival->debug->archival', async t => { + await testExportImport(t, A, D, A, 'snapshot', expectedDebugArtifacts); +}); +test('export and import data for state sync - archival->debug->debug', async t => { + await testExportImport(t, A, D, D, 'snapshot', expectedDebugArtifacts); +}); + +// ------------------------------------------------------------ + +test('export and import data for state sync - debug->current->current', async t => { + await testExportImport(t, D, C, C, 'none', expectedCurrentArtifacts); +}); +test('export and import data for state sync - debug->current->archival', async t => { + await testExportImport(t, D, C, A, 'none', expectedCurrentArtifacts); +}); +test('export and import data for state sync - debug->current->debug', async t => { + await testExportImport(t, D, C, D, 'none', expectedCurrentArtifacts); +}); + +test('export and import data for state sync - debug->archival->current', async t => { + await testExportImport(t, D, A, C, 'none', expectedArchivalArtifacts); +}); +test('export and import data for state sync - debug->archival->archival', async t => { + await testExportImport(t, D, A, A, 'none', expectedArchivalArtifacts); +}); +test('export and import data for state sync - debug->archival->debug', async t => { + await testExportImport(t, D, A, D, 'none', expectedArchivalArtifacts); +}); + +test('export and import data for state sync - debug->debug->current', async t => { + await testExportImport(t, D, D, C, 'none', expectedDebugArtifacts); +}); +test('export and import data for state sync - debug->debug->archival', async t => { + await testExportImport(t, D, D, A, 'none', expectedDebugArtifacts); +}); +test('export and import data for state sync - debug->debug->debug', async t => { + await testExportImport(t, D, D, D, 'none', expectedDebugArtifacts); +}); diff --git a/packages/swing-store/test/test-snapstore.js b/packages/swing-store/test/test-snapstore.js index 47f59f54945..fa3b64f8bbc 100644 --- a/packages/swing-store/test/test-snapstore.js +++ b/packages/swing-store/test/test-snapstore.js @@ -66,6 +66,7 @@ test('build temp file; compress to cache file', async t => { const exportInfo = { endPos: 47, hash: expectedHash, + inUse: 1, }; t.deepEqual(snapshotInfo, dbInfo); t.is(store.hasHash('fakeVatID', hash), true); diff --git a/packages/swing-store/test/test-state.js b/packages/swing-store/test/test-state.js index fc53d03efea..c2629428aff 100644 --- a/packages/swing-store/test/test-state.js +++ b/packages/swing-store/test/test-state.js @@ -151,6 +151,7 @@ async function testTranscriptStore(t, dbDir) { const exportLog = makeExportLog(); const { kernelStorage, hostStorage } = initSwingStore(dbDir, { exportCallback: exportLog.callback, + keepTranscripts: true, // XXX need to vary }); const { transcriptStore } = kernelStorage; const { commit, close } = hostStorage; diff --git a/packages/swing-store/test/test-stateSync.js b/packages/swing-store/test/test-stateSync.js deleted file mode 100644 index a24b972efaf..00000000000 --- a/packages/swing-store/test/test-stateSync.js +++ /dev/null @@ -1,232 +0,0 @@ -// @ts-check - -import '@endo/init/debug.js'; -import fs from 'fs'; - -// eslint-disable-next-line import/no-extraneous-dependencies -import test from 'ava'; -// eslint-disable-next-line import/no-extraneous-dependencies -import tmp from 'tmp'; - -import { - initSwingStore, - makeSwingStoreExporter, - importSwingStore, -} from '../src/swingStore.js'; - -function makeExportLog() { - const exportLog = []; - const shadowStore = new Map(); - return { - callback(updates) { - exportLog.push(updates); - for (const [key, value] of updates) { - if (value == null) { - shadowStore.delete(key); - } else { - shadowStore.set(key, value); - } - } - }, - getLog() { - return exportLog; - }, - entries() { - return shadowStore.entries(); - }, - }; -} - -/** - * @param {string} [prefix] - * @returns {Promise<[string, () => void]>} - */ -const tmpDir = prefix => - new Promise((resolve, reject) => { - tmp.dir({ unsafeCleanup: true, prefix }, (err, name, removeCallback) => { - if (err) { - reject(err); - } else { - resolve([name, removeCallback]); - } - }); - }); - -function actLikeAVatRunningACrank(vat, ks, crank) { - const { kvStore, transcriptStore } = ks; - const { vatID } = vat; - ks.startCrank(); - kvStore.set('kval', `set in ${crank}`); - kvStore.set(`${vatID}.vval`, `stuff in ${crank}`); - kvStore.set(`${vatID}.monotonic.${crank}`, 'more and more'); - if (crank % 3 === 0) { - kvStore.delete('brigadoon'); - } else { - kvStore.set('brigadoon', `here during ${crank}`); - } - ks.endCrank(); - transcriptStore.addItem(vatID, `stuff done during crank #${crank}`); - vat.endPos += 1; -} - -async function fakeAVatSnapshot(vat, ks) { - await ks.snapStore.saveSnapshot(vat.vatID, vat.endPos, async filePath => { - fs.writeFileSync( - filePath, - `snapshot of vat ${vat.vatID} as of ${vat.endPos}`, - ); - }); - ks.transcriptStore.rolloverSpan(vat.vatID); -} - -test('export and import data for state sync', async t => { - const exportLog = makeExportLog(); - const [dbDir, cleanup] = await tmpDir('testdb'); - t.teardown(cleanup); - const ssOut = initSwingStore(dbDir, { - exportCallback: exportLog.callback, - keepSnapshots: true, - }); - const { kernelStorage, debug } = ssOut; - - const vats = [ - { vatID: 'vatA', endPos: 0 }, - { vatID: 'vatB', endPos: 0 }, - ]; - for (const vat of vats) { - kernelStorage.transcriptStore.initTranscript(vat.vatID); - } - - // Run 4 "blocks", each consisting of 5 "cranks", across 2 "vats" - // Snapshot 'vatA' after the first and third blocks and 'vatB' after the second - // This will leave 2 current snapshots and 1 historical snapshot - let crankNum = 0; - for (let block = 0; block < 4; block += 1) { - for (let crank = 0; crank < 4; crank += 1) { - crankNum += 1; - const vat = vats[crankNum % vats.length]; - actLikeAVatRunningACrank(vat, kernelStorage, crankNum); - } - if (block < 3) { - // eslint-disable-next-line no-await-in-loop - await fakeAVatSnapshot(vats[block % 2], kernelStorage); - } - ssOut.hostStorage.commit(); - } - - const exporter = makeSwingStoreExporter(dbDir); - - const compareElems = (a, b) => a[0].localeCompare(b[0]); - - const kvData = []; - for await (const elem of exporter.getExportData()) { - kvData.push(elem); - } - kvData.sort(compareElems); - - const feedKVData = []; - for (const elem of exportLog.entries()) { - feedKVData.push(elem); - } - feedKVData.sort(compareElems); - - t.deepEqual(kvData, feedKVData); - t.deepEqual(kvData, [ - ['kv.brigadoon', 'here during 16'], - ['kv.kval', 'set in 16'], - ['kv.vatA.monotonic.10', 'more and more'], - ['kv.vatA.monotonic.12', 'more and more'], - ['kv.vatA.monotonic.14', 'more and more'], - ['kv.vatA.monotonic.16', 'more and more'], - ['kv.vatA.monotonic.2', 'more and more'], - ['kv.vatA.monotonic.4', 'more and more'], - ['kv.vatA.monotonic.6', 'more and more'], - ['kv.vatA.monotonic.8', 'more and more'], - ['kv.vatA.vval', 'stuff in 16'], - ['kv.vatB.monotonic.1', 'more and more'], - ['kv.vatB.monotonic.11', 'more and more'], - ['kv.vatB.monotonic.13', 'more and more'], - ['kv.vatB.monotonic.15', 'more and more'], - ['kv.vatB.monotonic.3', 'more and more'], - ['kv.vatB.monotonic.5', 'more and more'], - ['kv.vatB.monotonic.7', 'more and more'], - ['kv.vatB.monotonic.9', 'more and more'], - ['kv.vatB.vval', 'stuff in 15'], - [ - 'snapshot.vatA.2', - '{"vatID":"vatA","endPos":2,"hash":"6c7e452ee3eaec849c93234d933af4300012e4ff161c328d3c088ec3deef76a6"}', - ], - [ - 'snapshot.vatA.6', - '{"vatID":"vatA","endPos":6,"hash":"36afc9e2717c395759e308c4a877d491f967e9768d73520bde758ff4fac5d8b9"}', - ], - ['snapshot.vatA.current', 'snapshot.vatA.6'], - [ - 'snapshot.vatB.4', - '{"vatID":"vatB","endPos":4,"hash":"afd477014db678fbc1aa58beab50f444deb653b8cc8e8583214a363fd12ed57a"}', - ], - ['snapshot.vatB.current', 'snapshot.vatB.4'], - [ - 'transcript.vatA.0', - '{"vatID":"vatA","startPos":0,"endPos":2,"hash":"ea8ac1a751712ad66e4a9182adc65afe9bb0f4cd0ee0b828c895c63fbd2e3157","isCurrent":0}', - ], - [ - 'transcript.vatA.2', - '{"vatID":"vatA","startPos":2,"endPos":6,"hash":"88f299ca67b8acdf6023a83bb8e899af5adcf3271c7a1a2a495dcd6f1fbaac9f","isCurrent":0}', - ], - [ - 'transcript.vatA.current', - '{"vatID":"vatA","startPos":6,"endPos":8,"hash":"fe5d692b24a32d53bf617ba9ed3391b60c36a402c70a07a6aa984fc316e4efcc","isCurrent":1}', - ], - [ - 'transcript.vatB.0', - '{"vatID":"vatB","startPos":0,"endPos":4,"hash":"41dbf60cdec066106c7030517cb9f9f34a50fe2259705cf5fdbdd0b39ae12e46","isCurrent":0}', - ], - [ - 'transcript.vatB.current', - '{"vatID":"vatB","startPos":4,"endPos":8,"hash":"34fa09207bfb7af5fc3e65acb07f13b60834d0fbd2c6b9708f794c4397bd865d","isCurrent":1}', - ], - ]); - - const artifactNamesAll = []; - for await (const name of exporter.getArtifactNames(true)) { - artifactNamesAll.push(name); - } - t.deepEqual(artifactNamesAll, [ - 'snapshot.vatA.6', - 'snapshot.vatB.4', - 'snapshot.vatA.2', - 'transcript.vatA.0.2', - 'transcript.vatA.2.6', - 'transcript.vatA.6.8', - 'transcript.vatB.0.4', - 'transcript.vatB.4.8', - ]); - - const artifactNamesCurr = []; - for await (const name of exporter.getArtifactNames(false)) { - artifactNamesCurr.push(name); - } - t.deepEqual(artifactNamesCurr, [ - 'snapshot.vatA.6', - 'snapshot.vatB.4', - 'transcript.vatA.6.8', - 'transcript.vatB.4.8', - ]); - - const beforeDumpFull = debug.dump(true); - const ssInFull = await importSwingStore(exporter, null, { - includeHistorical: true, - }); - ssInFull.hostStorage.commit(); - const afterDumpFull = ssInFull.debug.dump(true); - t.deepEqual(beforeDumpFull, afterDumpFull); - - const beforeDumpCurr = debug.dump(false); - const ssInCurr = await importSwingStore(exporter, null, { - includeHistorical: false, - }); - ssInCurr.hostStorage.commit(); - const afterDumpCurr = ssInCurr.debug.dump(false); - t.deepEqual(beforeDumpCurr, afterDumpCurr); -});