Skip to content

Commit

Permalink
Merge branch 'master' into markm-store-provide
Browse files Browse the repository at this point in the history
  • Loading branch information
@mergify
mergify[bot] authored May 4, 2022
2 parents 2b94064 + da13a2b commit 915de5e
Show file tree
Hide file tree
Showing 7 changed files with 116 additions and 52 deletions.
5 changes: 5 additions & 0 deletions packages/SwingSet/src/liveslots/vatstore-usage.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,11 @@ Durable Kinds are defined just like virtual Kinds, but they use a different cons
The KindHandle is a durable virtual object of a special internal Kind. This is the first Kind allocated, so it generally gets Kind ID "1", and the handles will get vrefs of `o+1/N`.


# Kind Metadata

For each virtual object kind that is defined we store a metadata record for purposes of scanning directly through the defined kinds when a vat is stopped or upgraded. For durable kinds this record is stored in `vom.dkind.${kindID}`; for non-durable kinds it is stored in `vom.vkind.${kindID}`. Currently this metadata takes the form of a JSON-serialized record `{ kindID, tag }`, where the `kindID` property is the kind ID (redundantly) and `tag` is the tag string as provided in the `defineKind` or `makeKindHandle` call.


# Virtual/Durable Collections (aka Stores)

Liveslots provides a handful of "virtual collection" types to vats, to store high-cardinality data on disk rather than in RAM. These are also known as a `Store`. They provide limited range queries and offer a single fixed sort index: numbers sort as usual, BigInts sort as usual but separate from numbers, strings sort lexicographically by UTF-8 encoding, and object references sort by insertion order).
Expand Down
10 changes: 6 additions & 4 deletions packages/SwingSet/src/liveslots/virtualObjectManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,7 @@ export function makeVirtualObjectManager(

function defineKind(tag, init, behavior, options) {
const kindID = `${allocateExportID()}`;
syscall.vatstoreSet(`vom.vkind.${kindID}`, JSON.stringify({ kindID, tag }));
return defineKindInternal(
kindID,
tag,
Expand All @@ -759,6 +760,7 @@ export function makeVirtualObjectManager(

function defineKindMulti(tag, init, behavior, options) {
const kindID = `${allocateExportID()}`;
syscall.vatstoreSet(`vom.vkind.${kindID}`, JSON.stringify({ kindID, tag }));
return defineKindInternal(
kindID,
tag,
Expand All @@ -785,7 +787,7 @@ export function makeVirtualObjectManager(

function reanimateDurableKindID(vobjID) {
const { subid: kindID } = parseVatSlot(vobjID);
const raw = syscall.vatstoreGet(`vom.kind.${kindID}`);
const raw = syscall.vatstoreGet(`vom.dkind.${kindID}`);
assert(raw, X`unknown kind ID ${kindID}`);
const durableKindDescriptor = harden(JSON.parse(raw));
const kindHandle = Far('kind', {});
Expand All @@ -806,7 +808,7 @@ export function makeVirtualObjectManager(
kindDescriptors.set(kindHandle, durableKindDescriptor);
registerValue(kindIDvref, kindHandle, false);
syscall.vatstoreSet(
`vom.kind.${kindID}`,
`vom.dkind.${kindID}`,
JSON.stringify(durableKindDescriptor),
);
return kindHandle;
Expand Down Expand Up @@ -847,9 +849,9 @@ export function makeVirtualObjectManager(
}

function insistAllDurableKindsReconnected() {
// identify all user-defined durable kinds by iterating `vom.kind.*`
// identify all user-defined durable kinds by iterating `vom.dkind.*`
const missing = [];
const prefix = 'vom.kind.';
const prefix = 'vom.dkind.';
let [key, value] = syscall.vatstoreGetAfter('', prefix);
while (key) {
const descriptor = JSON.parse(value);
Expand Down
2 changes: 1 addition & 1 deletion packages/SwingSet/test/stores/test-storeGC/gc-helpers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -530,7 +530,7 @@ export function validateInit(v) {
validate(v, matchVatstoreGet('deadPromises', NONE));
validate(v, matchVatstoreDelete('deadPromises'));
validate(v, matchVatstoreGetAfter('', 'vc.3.', 'vc.3.{', [NONE, NONE]));
validate(v, matchVatstoreGetAfter('', 'vom.kind.', NONE, [NONE, NONE]));
validate(v, matchVatstoreGetAfter('', 'vom.dkind.', NONE, [NONE, NONE]));
}

export function validateDropHeld(v, rp, rc, es) {
Expand Down
2 changes: 1 addition & 1 deletion packages/SwingSet/test/test-baggage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ test.serial('exercise baggage', async t => {
validate(v, matchVatstoreGet('deadPromises', NONE));
validate(v, matchVatstoreDelete('deadPromises'));
validate(v, matchVatstoreGetAfter('', 'vc.3.', 'vc.3.{', [NONE, NONE]));
validate(v, matchVatstoreGetAfter('', 'vom.kind.', NONE, [NONE, NONE]));
validate(v, matchVatstoreGetAfter('', 'vom.dkind.', NONE, [NONE, NONE]));
validateDone(v);

const rp = await dispatchMessage('doSomething', capargs([]));
Expand Down
1 change: 1 addition & 0 deletions packages/SwingSet/test/virtualObjects/test-representatives.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ test('virtual object gc', async t => {
'v1.vs.vom.rc.o+6/1': '1',
'v1.vs.vom.rc.o+6/2': '1',
'v1.vs.vom.rc.o+6/3': '1',
'v1.vs.vom.vkind.10': '{"kindID":"10","tag":"thing"}',
'v1.vs.watchedPromiseTableID': 'o+6/3',
'v1.vs.watcherTableID': 'o+6/2',
});
Expand Down
105 changes: 62 additions & 43 deletions packages/SwingSet/test/virtualObjects/test-virtualObjectGC.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,14 +169,19 @@ function stateKey(vref) {
return `vom.${base(vref)}`;
}

const unfacetedThingKindID = 'o+10';
const facetedThingKindID = 'o+11';
const holderKindID = 'o+12';
const unfacetedThingKindID = '10';
const unfacetedThingBaseRef = `o+${unfacetedThingKindID}`;
const facetedThingKindID = '11';
const facetedThingBaseRef = `o+${facetedThingKindID}`;
const holderKindID = '12';
const holderBaseRef = `o+${holderKindID}`;
const markerKindID = '13';
const markerBaseRef = `o+${markerKindID}`;

const remotableID = 'o+14';

function thingVref(isf, instance) {
return `${isf ? facetedThingKindID : unfacetedThingKindID}/${instance}`;
return `${isf ? facetedThingBaseRef : unfacetedThingBaseRef}/${instance}`;
}

function facetRef(isf, vref, facet) {
Expand All @@ -185,7 +190,7 @@ function facetRef(isf, vref, facet) {

const cacheDisplacerVref = thingVref(false, 1);
const fCacheDisplacerVref = thingVref(true, 1);
const virtualHolderVref = `${holderKindID}/1`;
const virtualHolderVref = `${holderBaseRef}/1`;

const stringSchema = JSON.stringify(
capargs([{ '@qclass': 'tagged', tag: 'match:kind', payload: 'string' }]),
Expand Down Expand Up @@ -235,7 +240,7 @@ function buildRootObject(vatPowers) {
const virtualHolder = makeVirtualHolder();

const makeDualMarkerThing = defineKindMulti(
'thing',
'marker',
() => ({ unused: 'uncared for' }),
{
facetA: {
Expand Down Expand Up @@ -498,6 +503,16 @@ function validateCreateBuiltInTables(v) {
validateCreateWatchedPromiseTable(v, 3);
}

function validateKindMetadata(v, kindID, tag) {
validate(
v,
matchVatstoreSet(
`vom.vkind.${kindID}`,
`{"kindID":"${kindID}","tag":"${tag}"}`,
),
);
}

function validateSetup(v) {
validate(v, matchVatstoreGet('idCounters', NONE));
validate(v, matchVatstoreGet('kindIDID', NONE));
Expand All @@ -511,12 +526,16 @@ function validateSetup(v) {
),
);
validateCreateBuiltInTables(v);
validateKindMetadata(v, unfacetedThingKindID, 'thing');
validateKindMetadata(v, facetedThingKindID, 'thing');
validate(v, matchVatstoreSet(stateKey(cacheDisplacerVref), cacheObjValue));
validateKindMetadata(v, holderKindID, 'holder');
validate(v, matchVatstoreSet(stateKey(fCacheDisplacerVref), cacheObjValue));
validateKindMetadata(v, markerKindID, 'marker');
validate(v, matchVatstoreGet('deadPromises', NONE));
validate(v, matchVatstoreDelete('deadPromises'));
validate(v, matchVatstoreGetAfter('', 'vc.3.', 'vc.3.{', [NONE, NONE]));
validate(v, matchVatstoreGetAfter('', 'vom.kind.', NONE, [NONE, NONE]));
validate(v, matchVatstoreGetAfter('', 'vom.dkind.', NONE, [NONE, NONE]));
validate(
v,
matchVatstoreSet(stateKey(virtualHolderVref), heldThingValue(null)),
Expand Down Expand Up @@ -1080,7 +1099,7 @@ test.serial('VO multifacet export 1', async t => {
'bob',
true,
);
const thing = `${facetedThingKindID}/2`;
const thing = `${facetedThingBaseRef}/2`;
const thingf = `${thing}:0`;

// lerv -> Lerv Create facets
Expand All @@ -1102,7 +1121,7 @@ test.serial('VO multifacet export 2a', async t => {
'bob',
true,
);
const thing = `${facetedThingKindID}/2`;
const thing = `${facetedThingBaseRef}/2`;
const thingA = `${thing}:0`;

// lerv -> Lerv Create facets
Expand Down Expand Up @@ -1130,7 +1149,7 @@ test.serial('VO multifacet export 2b', async t => {
'bob',
true,
);
const thing = `${facetedThingKindID}/2`;
const thing = `${facetedThingBaseRef}/2`;
const thingB = `${thing}:1`;

// lerv -> Lerv Create facets
Expand Down Expand Up @@ -1158,7 +1177,7 @@ test.serial('VO multifacet export 3abba', async t => {
'bob',
true,
);
const thing = `${facetedThingKindID}/2`;
const thing = `${facetedThingBaseRef}/2`;
const thingA = `${thing}:0`;
const thingB = `${thing}:1`;

Expand Down Expand Up @@ -1195,7 +1214,7 @@ test.serial('VO multifacet export 3abab', async t => {
'bob',
true,
);
const thing = `${facetedThingKindID}/2`;
const thing = `${facetedThingBaseRef}/2`;
const thingA = `${thing}:0`;
const thingB = `${thing}:1`;

Expand Down Expand Up @@ -1231,8 +1250,8 @@ test.serial('VO multifacet markers only', async t => {
'bob',
true,
);
const thing = 'o+13/1';
const thingf = 'o+13/1:0';
const thing = `${markerBaseRef}/1`;
const thingf = `${markerBaseRef}/1:0`;
const thingCapdata = JSON.stringify({ unused: capdata('uncared for') });

// lerv -> Lerv Create facets
Expand All @@ -1251,11 +1270,11 @@ function validatePrepareStore3(v, rp, isf) {
validate(v, matchVatstoreSet(rcKey(thing), '1'));
validate(v, matchVatstoreGet(rcKey(thing), '1'));
validate(v, matchVatstoreSet(rcKey(thing), '2'));
validate(v, matchVatstoreSet(stateKey(`${holderKindID}/2`), heldThingValue(thing)));
validate(v, matchVatstoreSet(stateKey(`${holderBaseRef}/2`), heldThingValue(thing)));
validate(v, matchVatstoreGet(rcKey(thing), '2'));
validate(v, matchVatstoreSet(rcKey(thing), '3'));
validate(v, matchVatstoreSet(stateKey(`${holderKindID}/3`), heldThingValue(thing)));
validate(v, matchVatstoreSet(stateKey(`${holderKindID}/4`), heldThingValue(thing)));
validate(v, matchVatstoreSet(stateKey(`${holderBaseRef}/3`), heldThingValue(thing)));
validate(v, matchVatstoreSet(stateKey(`${holderBaseRef}/4`), heldThingValue(thing)));
validate(v, matchVatstoreGet(stateKey(cacheDisplacerVref), cacheObjValue));
validateReturned(v, rp);
validate(v, matchVatstoreGet(rcKey(thing), '3'));
Expand All @@ -1276,18 +1295,18 @@ async function voRefcountManagementTest1(t, isf) {
validatePrepareStore3(v, rp, isf);

rp = await dispatchMessage('finishClearHolders');
validate(v, matchVatstoreGet(stateKey(`${holderKindID}/2`), heldThingValue(thingf)));
validate(v, matchVatstoreGet(stateKey(`${holderBaseRef}/2`), heldThingValue(thingf)));
validate(v, matchVatstoreGet(rcKey(thing), '3'));
validate(v, matchVatstoreSet(rcKey(thing), '2'));
validate(v, matchVatstoreSet(stateKey(`${holderKindID}/2`), heldThingValue(null)));
validate(v, matchVatstoreGet(stateKey(`${holderKindID}/3`), heldThingValue(thingf)));
validate(v, matchVatstoreSet(stateKey(`${holderBaseRef}/2`), heldThingValue(null)));
validate(v, matchVatstoreGet(stateKey(`${holderBaseRef}/3`), heldThingValue(thingf)));
validate(v, matchVatstoreGet(rcKey(thing), '2'));
validate(v, matchVatstoreSet(rcKey(thing), '1'));
validate(v, matchVatstoreSet(stateKey(`${holderKindID}/3`), heldThingValue(null)));
validate(v, matchVatstoreGet(stateKey(`${holderKindID}/4`), heldThingValue(thingf)));
validate(v, matchVatstoreSet(stateKey(`${holderBaseRef}/3`), heldThingValue(null)));
validate(v, matchVatstoreGet(stateKey(`${holderBaseRef}/4`), heldThingValue(thingf)));
validate(v, matchVatstoreGet(rcKey(thing), '1'));
validate(v, matchVatstoreSet(rcKey(thing), '0'));
validate(v, matchVatstoreSet(stateKey(`${holderKindID}/4`), heldThingValue(null)));
validate(v, matchVatstoreSet(stateKey(`${holderBaseRef}/4`), heldThingValue(null)));
validate(v, matchVatstoreGet(stateKey(cacheDisplacerVref), cacheObjValue));
validateReturned(v, rp);
validateStatusCheck(v, thing, '0', NONE, testObjValue);
Expand Down Expand Up @@ -1317,21 +1336,21 @@ async function voRefcountManagementTest2(t, isf) {
rp = await dispatchMessage('finishDropHolders');
validateReturned(v, rp);

const holder2 = `${holderKindID}/2`;
const holder2 = `${holderBaseRef}/2`;
validateStatusCheck(v, holder2, NONE, NONE, heldThingValue(thingf));
validate(v, matchVatstoreGet(rcKey(thing), '3'));
validate(v, matchVatstoreSet(rcKey(thing), '2'));
validateDelete(v, holder2);
validateWeakCheck(v, holder2);

const holder3 = `${holderKindID}/3`;
const holder3 = `${holderBaseRef}/3`;
validateStatusCheck(v, holder3, NONE, NONE, heldThingValue(thingf));
validate(v, matchVatstoreGet(rcKey(thing), '2'));
validate(v, matchVatstoreSet(rcKey(thing), '1'));
validateDelete(v, holder3);
validateWeakCheck(v, holder3);

const holder4 = `${holderKindID}/4`;
const holder4 = `${holderBaseRef}/4`;
validateStatusCheck(v, holder4, NONE, NONE, heldThingValue(thingf));
validate(v, matchVatstoreGet(rcKey(thing), '1'));
validate(v, matchVatstoreSet(rcKey(thing), '0'));
Expand Down Expand Up @@ -1363,15 +1382,15 @@ async function voRefcountManagementTest3(t, isf) {
rp = await dispatchMessage('prepareStoreLinked');
validate(v, matchVatstoreGet(rcKey(thing)));
validate(v, matchVatstoreSet(rcKey(thing), '1'));
const holder2 = `${holderKindID}/2`;
const holder2 = `${holderBaseRef}/2`;
validate(v, matchVatstoreGet(rcKey(holder2)));
validate(v, matchVatstoreSet(rcKey(holder2), '1'));
validate(v, matchVatstoreSet(stateKey(holder2), heldThingValue(thingf)));
const holder3 = `${holderKindID}/3`;
const holder3 = `${holderBaseRef}/3`;
validate(v, matchVatstoreGet(rcKey(holder3)));
validate(v, matchVatstoreSet(rcKey(holder3), '1'));
validate(v, matchVatstoreSet(stateKey(holder3), heldHolderValue(holder2)));
const holder4 = `${holderKindID}/4`;
const holder4 = `${holderBaseRef}/4`;
validate(v, matchVatstoreSet(stateKey(holder4), heldHolderValue(holder3)));
validate(v, matchVatstoreGet(stateKey(cacheDisplacerVref), cacheObjValue));
validateReturned(v, rp);
Expand Down Expand Up @@ -1435,13 +1454,13 @@ test.serial('presence refcount management 1', async t => {
validate(v, matchVatstoreSet(rcKey(presRef), '1'));
validate(v, matchVatstoreGet(rcKey(presRef), '1'));
validate(v, matchVatstoreSet(rcKey(presRef), '2'));
const holder2 = `${holderKindID}/2`;
const holder2 = `${holderBaseRef}/2`;
validate(v, matchVatstoreSet(stateKey(holder2), heldThingValue(presRef)));
validate(v, matchVatstoreGet(rcKey(presRef), '2'));
validate(v, matchVatstoreSet(rcKey(presRef), '3'));
const holder3 = `${holderKindID}/3`;
const holder3 = `${holderBaseRef}/3`;
validate(v, matchVatstoreSet(stateKey(holder3), heldThingValue(presRef)));
validate(v, matchVatstoreSet(stateKey(`${holderKindID}/4`), heldThingValue(presRef)));
validate(v, matchVatstoreSet(stateKey(`${holderBaseRef}/4`), heldThingValue(presRef)));
validate(v, matchVatstoreGet(stateKey(cacheDisplacerVref), cacheObjValue));
validateReturned(v, rp);
validate(v, matchVatstoreGet(rcKey(presRef), '3'));
Expand All @@ -1456,11 +1475,11 @@ test.serial('presence refcount management 1', async t => {
validate(v, matchVatstoreGet(rcKey(presRef), '2'));
validate(v, matchVatstoreSet(rcKey(presRef), '1'));
validate(v, matchVatstoreSet(stateKey(holder3), heldThingValue(null)));
validate(v, matchVatstoreGet(stateKey(`${holderKindID}/4`), heldThingValue(presRef)));
validate(v, matchVatstoreGet(stateKey(`${holderBaseRef}/4`), heldThingValue(presRef)));
validate(v, matchVatstoreGet(rcKey(presRef), '1'));
validate(v, matchVatstoreSet(rcKey(presRef), '0'));
validate(v, matchVatstoreDelete(rcKey(presRef)));
validate(v, matchVatstoreSet(stateKey(`${holderKindID}/4`), heldThingValue(null)));
validate(v, matchVatstoreSet(stateKey(`${holderBaseRef}/4`), heldThingValue(null)));
validate(v, matchVatstoreGet(stateKey(cacheDisplacerVref), cacheObjValue));
validateReturned(v, rp);
validate(v, matchVatstoreGet(rcKey(presRef)));
Expand All @@ -1487,13 +1506,13 @@ test.serial('presence refcount management 2', async t => {
validate(v, matchVatstoreSet(rcKey(presRef), '1'));
validate(v, matchVatstoreGet(rcKey(presRef), '1'));
validate(v, matchVatstoreSet(rcKey(presRef), '2'));
const holder2 = `${holderKindID}/2`;
const holder2 = `${holderBaseRef}/2`;
validate(v, matchVatstoreSet(stateKey(holder2), heldThingValue(presRef)));
validate(v, matchVatstoreGet(rcKey(presRef), '2'));
validate(v, matchVatstoreSet(rcKey(presRef), '3'));
const holder3 = `${holderKindID}/3`;
const holder3 = `${holderBaseRef}/3`;
validate(v, matchVatstoreSet(stateKey(holder3), heldThingValue(presRef)));
const holder4 = `${holderKindID}/4`;
const holder4 = `${holderBaseRef}/4`;
validate(v, matchVatstoreSet(stateKey(holder4), heldThingValue(presRef)));
validate(v, matchVatstoreGet(stateKey(cacheDisplacerVref), cacheObjValue));
validateReturned(v, rp);
Expand Down Expand Up @@ -1537,11 +1556,11 @@ test.serial('remotable refcount management 1', async t => {
validateDone(v);

rp = await dispatchMessage('prepareStore3');
const holder2 = `${holderKindID}/2`;
const holder2 = `${holderBaseRef}/2`;
validate(v, matchVatstoreSet(stateKey(holder2), heldThingValue(remotableID)));
const holder3 = `${holderKindID}/3`;
const holder3 = `${holderBaseRef}/3`;
validate(v, matchVatstoreSet(stateKey(holder3), heldThingValue(remotableID)));
const holder4 = `${holderKindID}/4`;
const holder4 = `${holderBaseRef}/4`;
validate(v, matchVatstoreSet(stateKey(holder4), heldThingValue(remotableID)));
validate(v, matchVatstoreGet(stateKey(cacheDisplacerVref), cacheObjValue));
validateReturned(v, rp);
Expand Down Expand Up @@ -1572,11 +1591,11 @@ test.serial('remotable refcount management 2', async t => {
validateDone(v);

rp = await dispatchMessage('prepareStore3');
const holder2 = `${holderKindID}/2`;
const holder2 = `${holderBaseRef}/2`;
validate(v, matchVatstoreSet(stateKey(holder2), heldThingValue(remotableID)));
const holder3 = `${holderKindID}/3`;
const holder3 = `${holderBaseRef}/3`;
validate(v, matchVatstoreSet(stateKey(holder3), heldThingValue(remotableID)));
const holder4 = `${holderKindID}/4`;
const holder4 = `${holderBaseRef}/4`;
validate(v, matchVatstoreSet(stateKey(holder4), heldThingValue(remotableID)));
validate(v, matchVatstoreGet(stateKey(cacheDisplacerVref), cacheObjValue));
validateReturned(v, rp);
Expand Down
Loading

0 comments on commit 915de5e

Please sign in to comment.