diff --git a/packages/boot/test/orchestration/contract-upgrade.test.ts b/packages/boot/test/orchestration/contract-upgrade.test.ts new file mode 100644 index 00000000000..514652175ee --- /dev/null +++ b/packages/boot/test/orchestration/contract-upgrade.test.ts @@ -0,0 +1,78 @@ +/** @file Bootstrap test of restarting contracts using orchestration */ +import { test as anyTest } from '@agoric/zoe/tools/prepare-test-env-ava.js'; +import { TestFn } from 'ava'; + +import { + makeWalletFactoryContext, + type WalletFactoryTestContext, +} from '../bootstrapTests/walletFactory.js'; + +const test: TestFn = anyTest; +test.before(async t => { + t.context = await makeWalletFactoryContext( + t, + '@agoric/vm-config/decentral-itest-orchestration-config.json', + ); +}); +test.after.always(t => t.context.shutdown?.()); + +/** + * This test core-evals a buggy installation of the sendAnywhere contract by + * giving it a faulty `agoricNames` service with a lookup() function returns a + * promise that never resolves. + * + * Because the send-anywhere flow requires a lookup(), it waits forever. This + * gives us a point at which we can upgrade the vat with a working agoricNames + * and see that the flow continues from that point. + */ +test('resume', async t => { + const { walletFactoryDriver, buildProposal, evalProposal, storage } = + t.context; + + const { IST } = t.context.agoricNamesRemotes.brand; + + t.log('start sendAnywhere'); + await evalProposal( + buildProposal( + '@agoric/builders/scripts/testing/start-buggy-sendAnywhere.js', + ), + ); + + t.log('making offer'); + const wallet = await walletFactoryDriver.provideSmartWallet('agoric1test'); + // no money in wallet to actually send + const zero = { brand: IST, value: 0n }; + // send because it won't resolve + await wallet.sendOffer({ + id: 'send-somewhere', + invitationSpec: { + source: 'agoricContract', + instancePath: ['sendAnywhere'], + callPipe: [['makeSendInvitation']], + }, + proposal: { + // @ts-expect-error XXX BoardRemote + give: { Send: zero }, + }, + offerArgs: { destAddr: 'cosmos1whatever', chainName: 'cosmoshub' }, + }); + + // XXX golden test + const getLogged = () => + JSON.parse(storage.data.get('published.sendAnywhere.log')!).values; + + // This log shows the flow started, but didn't get past the name lookup + t.deepEqual(getLogged(), ['sending {0} from cosmoshub to cosmos1whatever']); + + t.log('upgrade sendAnywhere with fix'); + await evalProposal( + buildProposal('@agoric/builders/scripts/testing/fix-buggy-sendAnywhere.js'), + ); + + t.deepEqual(getLogged(), [ + 'sending {0} from cosmoshub to cosmos1whatever', + // XXX this denom list may be wrong + 'got info for denoms: ubld, uist', + 'transfer complete, seat exited', + ]); +}); diff --git a/packages/builders/scripts/testing/fix-buggy-sendAnywhere.js b/packages/builders/scripts/testing/fix-buggy-sendAnywhere.js new file mode 100644 index 00000000000..fdb31f11d44 --- /dev/null +++ b/packages/builders/scripts/testing/fix-buggy-sendAnywhere.js @@ -0,0 +1,142 @@ +/** + * @file This is for use in tests in a3p-integration + * Unlike most builder scripts, this one includes the proposal exports as well. + */ +import { + deeplyFulfilledObject, + makeTracer, + NonNullish, +} from '@agoric/internal'; +import { E, Far } from '@endo/far'; + +/// +/** + * @import {Installation, Instance} from '@agoric/zoe/src/zoeService/utils.js'; + */ + +const trace = makeTracer('FixBuggySA', true); + +/** + * @import {start as StartFn} from '@agoric/orchestration/src/examples/send-anywhere.contract.js'; + */ + +/** + * @param {BootstrapPowers & { + * instance: { + * consume: { + * sendAnywhere: Instance; + * }; + * }; + * }} powers + * @param {...any} rest + */ +export const fixSendAnywhere = async ( + { + consume: { + agoricNames, + board, + chainStorage, + chainTimerService, + contractKits, + cosmosInterchainService, + localchain, + }, + instance: instances, + }, + { options: { sendAnywhereRef } }, +) => { + trace(fixSendAnywhere.name); + + const saInstance = await instances.consume.sendAnywhere; + trace('saInstance', saInstance); + const saKit = await E(contractKits).get(saInstance); + + const marshaller = await E(board).getReadonlyMarshaller(); + + // This apparently pointless wrapper is to maintain structural parity + // with the buggy core-eval's wrapper to make lookup() hang. + const agoricNamesResolves = Far('agoricNames that resolves', { + lookup: async (...args) => { + return E(agoricNames).lookup(...args); + }, + }); + + const privateArgs = await deeplyFulfilledObject( + harden({ + agoricNames: agoricNamesResolves, + localchain, + marshaller, + orchestrationService: cosmosInterchainService, + storageNode: E(NonNullish(await chainStorage)).makeChildNode( + 'sendAnywhere', + ), + timerService: chainTimerService, + }), + ); + + trace('upgrading...'); + await E(saKit.adminFacet).upgradeContract( + sendAnywhereRef.bundleID, + privateArgs, + ); + + trace('done'); +}; +harden(fixSendAnywhere); + +export const getManifestForValueVow = ({ restoreRef }, { sendAnywhereRef }) => { + console.log('sendAnywhereRef', sendAnywhereRef); + return { + manifest: { + [fixSendAnywhere.name]: { + consume: { + agoricNames: true, + board: true, + chainStorage: true, + chainTimerService: true, + cosmosInterchainService: true, + localchain: true, + + contractKits: true, + }, + installation: { + consume: { sendAnywhere: true }, + }, + instance: { + consume: { sendAnywhere: true }, + }, + }, + }, + installations: { + sendAnywhere: restoreRef(sendAnywhereRef), + }, + options: { + sendAnywhereRef, + }, + }; +}; + +/** @type {import('@agoric/deploy-script-support/src/externalTypes.js').CoreEvalBuilder} */ +export const defaultProposalBuilder = async ({ publishRef, install }) => + harden({ + // Somewhat unorthodox, source the exports from this builder module + sourceSpec: '@agoric/builders/scripts/testing/fix-buggy-sendAnywhere.js', + getManifestCall: [ + 'getManifestForValueVow', + { + sendAnywhereRef: publishRef( + install( + '@agoric/orchestration/src/examples/send-anywhere.contract.js', + ), + ), + }, + ], + }); + +export default async (homeP, endowments) => { + // import dynamically so the module can work in CoreEval environment + const dspModule = await import('@agoric/deploy-script-support'); + const { makeHelpers } = dspModule; + const { writeCoreEval } = await makeHelpers(homeP, endowments); + await writeCoreEval(fixSendAnywhere.name, defaultProposalBuilder); +}; diff --git a/packages/builders/scripts/testing/start-buggy-sendAnywhere.js b/packages/builders/scripts/testing/start-buggy-sendAnywhere.js new file mode 100644 index 00000000000..6bfb9933643 --- /dev/null +++ b/packages/builders/scripts/testing/start-buggy-sendAnywhere.js @@ -0,0 +1,140 @@ +/** + * @file This is for use in tests in a3p-integration + * Unlike most builder scripts, this one includes the proposal exports as well. + */ +import { + deeplyFulfilledObject, + makeTracer, + NonNullish, +} from '@agoric/internal'; +import { E, Far } from '@endo/far'; + +/// +/** + * @import {Installation} from '@agoric/zoe/src/zoeService/utils.js'; + */ + +const trace = makeTracer('StartBuggySA', true); + +/** + * @import {start as StartFn} from '@agoric/orchestration/src/examples/send-anywhere.contract.js'; + */ + +/** + * @param {BootstrapPowers & { + * installation: { + * consume: { + * sendAnywhere: Installation; + * }; + * }; + * }} powers + */ +export const startSendAnywhere = async ({ + consume: { + agoricNames, + board, + chainStorage, + chainTimerService, + cosmosInterchainService, + localchain, + startUpgradable, + }, + installation: { + consume: { sendAnywhere }, + }, + instance: { + // @ts-expect-error unknown instance + produce: { sendAnywhere: produceInstance }, + }, +}) => { + trace(startSendAnywhere.name); + + const marshaller = await E(board).getReadonlyMarshaller(); + + const privateArgs = await deeplyFulfilledObject( + harden({ + agoricNames, + localchain, + marshaller, + orchestrationService: cosmosInterchainService, + storageNode: E(NonNullish(await chainStorage)).makeChildNode( + 'sendAnywhere', + ), + timerService: chainTimerService, + }), + ); + + const agoricNamesHangs = Far('agoricNames that hangs', { + lookup: async () => { + trace('agoricNames.lookup being called that will never resolve'); + // BUG: this never resolves + return new Promise(() => {}); + }, + }); + + const { instance } = await E(startUpgradable)({ + label: 'sendAnywhere', + installation: sendAnywhere, + privateArgs: { + ...privateArgs, + agoricNames: agoricNamesHangs, + }, + }); + produceInstance.resolve(instance); + trace('done'); +}; +harden(startSendAnywhere); + +export const getManifestForValueVow = ({ restoreRef }, { sendAnywhereRef }) => { + trace('sendAnywhereRef', sendAnywhereRef); + return { + manifest: { + [startSendAnywhere.name]: { + consume: { + agoricNames: true, + board: true, + chainStorage: true, + chainTimerService: true, + cosmosInterchainService: true, + localchain: true, + + startUpgradable: true, + }, + installation: { + consume: { sendAnywhere: true }, + }, + instance: { + produce: { sendAnywhere: true }, + }, + }, + }, + installations: { + sendAnywhere: restoreRef(sendAnywhereRef), + }, + }; +}; + +/** @type {import('@agoric/deploy-script-support/src/externalTypes.js').CoreEvalBuilder} */ +export const defaultProposalBuilder = async ({ publishRef, install }) => + harden({ + // Somewhat unorthodox, source the exports from this builder module + sourceSpec: '@agoric/builders/scripts/testing/start-buggy-sendAnywhere.js', + getManifestCall: [ + 'getManifestForValueVow', + { + sendAnywhereRef: publishRef( + install( + '@agoric/orchestration/src/examples/send-anywhere.contract.js', + ), + ), + }, + ], + }); + +export default async (homeP, endowments) => { + // import dynamically so the module can work in CoreEval environment + const dspModule = await import('@agoric/deploy-script-support'); + const { makeHelpers } = dspModule; + const { writeCoreEval } = await makeHelpers(homeP, endowments); + await writeCoreEval(startSendAnywhere.name, defaultProposalBuilder); +}; diff --git a/packages/internal/src/upgrade-api.js b/packages/internal/src/upgrade-api.js index c2cadd7be16..a1b0fa9c640 100644 --- a/packages/internal/src/upgrade-api.js +++ b/packages/internal/src/upgrade-api.js @@ -45,5 +45,23 @@ harden(makeUpgradeDisconnection); * @returns {reason is UpgradeDisconnection} */ export const isUpgradeDisconnection = reason => - isFrozen(reason) && matches(reason, UpgradeDisconnectionShape); + reason != null && // eslint-disable-line eqeqeq + isFrozen(reason) && + matches(reason, UpgradeDisconnectionShape); harden(isUpgradeDisconnection); + +/** + * Returns whether a reason is a 'vat terminated' error generated when an object + * is abandoned by a vat during an upgrade. + * + * @param {any} reason + * @returns {reason is Error} + */ +export const isAbandonedError = reason => + reason != null && // eslint-disable-line eqeqeq + isFrozen(reason) && + matches(reason, M.error()) && + // We're not using a constant here since this special value is already + // sprinkled throughout the SDK + reason.message === 'vat terminated'; +harden(isAbandonedError); diff --git a/packages/internal/test/upgrade-api.test.js b/packages/internal/test/upgrade-api.test.js index 5ffc38f9b29..f8eee5d2d92 100644 --- a/packages/internal/test/upgrade-api.test.js +++ b/packages/internal/test/upgrade-api.test.js @@ -1,8 +1,11 @@ // @ts-check import test from 'ava'; +import { makeMarshal } from '@endo/marshal'; + import { makeUpgradeDisconnection, isUpgradeDisconnection, + isAbandonedError, } from '../src/upgrade-api.js'; test('isUpgradeDisconnection must recognize disconnection objects', t => { @@ -18,3 +21,15 @@ test('isUpgradeDisconnection must recognize original-format disconnection object }); t.true(isUpgradeDisconnection(disconnection)); }); + +test('isAbandonedError recognizes marshalled vat terminated errors', t => { + const { fromCapData, toCapData } = makeMarshal(undefined, undefined, { + serializeBodyFormat: 'smallcaps', + errorIdNum: 70_000, + marshalSaveError: () => {}, + }); + const error = harden(Error('vat terminated')); + const remoteError = fromCapData(toCapData(error)); + + t.true(isAbandonedError(remoteError)); +}); diff --git a/packages/orchestration/src/examples/send-anywhere.contract.js b/packages/orchestration/src/examples/send-anywhere.contract.js index ebe77f1af7d..40c5e434fc0 100644 --- a/packages/orchestration/src/examples/send-anywhere.contract.js +++ b/packages/orchestration/src/examples/send-anywhere.contract.js @@ -1,12 +1,15 @@ import { makeSharedStateRecord } from '@agoric/async-flow'; + import { AmountShape } from '@agoric/ertp'; import { InvitationShape } from '@agoric/zoe/src/typeGuards.js'; import { M } from '@endo/patterns'; +import { E } from '@endo/far'; import { withOrchestration } from '../utils/start-helper.js'; import * as flows from './send-anywhere.flows.js'; import { prepareChainHubAdmin } from '../exos/chain-hub-admin.js'; /** + * @import {Vow} from '@agoric/vow'; * @import {Zone} from '@agoric/zone'; * @import {OrchestrationPowers, OrchestrationTools} from '../utils/start-helper.js'; */ @@ -29,11 +32,11 @@ harden(SingleAmountRecord); * @param {Zone} zone * @param {OrchestrationTools} tools */ -const contract = async ( +export const contract = async ( zcf, privateArgs, zone, - { chainHub, orchestrateAll, zoeTools }, + { chainHub, orchestrateAll, vowTools, zoeTools }, ) => { const contractState = makeSharedStateRecord( /** @type {{ account: OrchestrationAccount | undefined }} */ { @@ -43,10 +46,16 @@ const contract = async ( const creatorFacet = prepareChainHubAdmin(zone, chainHub); + // UNTIL https://github.com/Agoric/agoric-sdk/issues/9066 + const logNode = E(privateArgs.storageNode).makeChildNode('log'); + /** @type {(msg: string) => Vow} */ + const log = msg => vowTools.watch(E(logNode).setValue(msg)); + // orchestrate uses the names on orchestrationFns to do a "prepare" of the associated behavior const orchFns = orchestrateAll(flows, { zcf, contractState, + log, localTransfer: zoeTools.localTransfer, }); @@ -69,6 +78,7 @@ const contract = async ( return { publicFacet, creatorFacet }; }; +harden(contract); export const start = withOrchestration(contract); harden(start); diff --git a/packages/orchestration/src/examples/send-anywhere.flows.js b/packages/orchestration/src/examples/send-anywhere.flows.js index 8573dea7b13..191109d025b 100644 --- a/packages/orchestration/src/examples/send-anywhere.flows.js +++ b/packages/orchestration/src/examples/send-anywhere.flows.js @@ -3,6 +3,7 @@ import { M, mustMatch } from '@endo/patterns'; /** * @import {GuestOf} from '@agoric/async-flow'; + * @import {Vow} from '@agoric/vow'; * @import {ZoeTools} from '../utils/zoe-tools.js'; * @import {Orchestrator, LocalAccountMethods, OrchestrationAccountI, OrchestrationFlow} from '../types.js'; */ @@ -18,12 +19,13 @@ const { entries } = Object; * @param {object} ctx * @param {{ localAccount?: OrchestrationAccountI & LocalAccountMethods }} ctx.contractState * @param {GuestOf} ctx.localTransfer + * @param {GuestOf<(msg: string) => Vow>} ctx.log * @param {ZCFSeat} seat * @param {{ chainName: string; destAddr: string }} offerArgs */ export const sendIt = async ( orch, - { contractState, localTransfer }, + { contractState, localTransfer, log }, seat, offerArgs, ) => { @@ -32,8 +34,10 @@ export const sendIt = async ( // NOTE the proposal shape ensures that the `give` is a single asset const { give } = seat.getProposal(); const [[_kw, amt]] = entries(give); + void log(`sending {${amt.value}} from ${chainName} to ${destAddr}`); const agoric = await orch.getChain('agoric'); const assets = await agoric.getVBankAssetInfo(); + void log(`got info for denoms: ${assets.map(a => a.denom).join(', ')}`); const { denom } = NonNullish( assets.find(a => a.brand === amt.brand), `${amt.brand} not registered in vbank`, @@ -41,15 +45,19 @@ export const sendIt = async ( const chain = await orch.getChain(chainName); if (!contractState.localAccount) { - const agoricChain = await orch.getChain('agoric'); - contractState.localAccount = await agoricChain.makeAccount(); + contractState.localAccount = await agoric.makeAccount(); } const info = await chain.getChainInfo(); const { chainId } = info; assert(typeof chainId === 'string', 'bad chainId'); - await localTransfer(seat, contractState.localAccount, give); + await localTransfer( + seat, + // @ts-expect-error Index signature for type 'string' is missing in type 'OrchestrationAccountI & LocalAccountMethods' + contractState.localAccount, + give, + ); await contractState.localAccount.transfer( { denom, value: amt.value }, @@ -60,5 +68,6 @@ export const sendIt = async ( }, ); seat.exit(); + void log(`transfer complete, seat exited`); }; harden(sendIt); diff --git a/packages/orchestration/src/examples/stakeBld.contract.js b/packages/orchestration/src/examples/stakeBld.contract.js index d55f82618f6..b3d53404aef 100644 --- a/packages/orchestration/src/examples/stakeBld.contract.js +++ b/packages/orchestration/src/examples/stakeBld.contract.js @@ -42,7 +42,11 @@ export const start = async (zcf, privateArgs, baggage) => { ); const vowTools = prepareVowTools(zone.subZone('vows')); - const chainHub = makeChainHub(privateArgs.agoricNames, vowTools); + const chainHub = makeChainHub( + zone.subZone('chainHub'), + privateArgs.agoricNames, + vowTools, + ); const { localchain, timerService } = privateArgs; const makeLocalOrchestrationAccountKit = prepareLocalOrchestrationAccountKit( diff --git a/packages/orchestration/src/examples/stakeIca.contract.js b/packages/orchestration/src/examples/stakeIca.contract.js index d6b89145368..11dde1662b2 100644 --- a/packages/orchestration/src/examples/stakeIca.contract.js +++ b/packages/orchestration/src/examples/stakeIca.contract.js @@ -84,7 +84,11 @@ export const start = async (zcf, privateArgs, baggage) => { const vowTools = prepareVowTools(zone.subZone('vows')); - const chainHub = makeChainHub(agoricNames, vowTools); + const chainHub = makeChainHub( + zone.subZone('chainHub'), + agoricNames, + vowTools, + ); const makeCosmosOrchestrationAccount = prepareCosmosOrchestrationAccount( zone, diff --git a/packages/orchestration/src/exos/chain-hub.js b/packages/orchestration/src/exos/chain-hub.js index f938771a480..0f4356d9777 100644 --- a/packages/orchestration/src/exos/chain-hub.js +++ b/packages/orchestration/src/exos/chain-hub.js @@ -4,12 +4,12 @@ import { M } from '@endo/patterns'; import { BrandShape } from '@agoric/ertp/src/typeGuards.js'; import { VowShape } from '@agoric/vow'; -import { makeHeapZone } from '@agoric/zone'; import { CosmosChainInfoShape, IBCConnectionInfoShape } from '../typeGuards.js'; /** * @import {NameHub} from '@agoric/vats'; * @import {Vow, VowTools} from '@agoric/vow'; + * @import {Zone} from '@agoric/zone'; * @import {CosmosAssetInfo, CosmosChainInfo, IBCConnectionInfo} from '../cosmos-api.js'; * @import {ChainInfo, KnownChains} from '../chain-info.js'; * @import {Denom} from '../orchestration-api.js'; @@ -174,18 +174,18 @@ const ChainHubI = M.interface('ChainHub', { }); /** - * Make a new ChainHub in the zone (or in the heap if no zone is provided). + * Make a new ChainHub in the zone. * * The resulting object is an Exo singleton. It has no precious state. It's only * state is a cache of queries to agoricNames and whatever info was provided in * registration calls. When you need a newer version you can simply make a hub * hub and repeat the registrations. * + * @param {Zone} zone * @param {Remote} agoricNames * @param {VowTools} vowTools */ -export const makeChainHub = (agoricNames, vowTools) => { - const zone = makeHeapZone(); +export const makeChainHub = (zone, agoricNames, vowTools) => { /** @type {MapStore} */ const chainInfos = zone.mapStore('chainInfos', { keyShape: M.string(), diff --git a/packages/orchestration/src/proposals/start-stakeAtom.js b/packages/orchestration/src/proposals/start-stakeAtom.js index 838bacfd832..cab85042813 100644 --- a/packages/orchestration/src/proposals/start-stakeAtom.js +++ b/packages/orchestration/src/proposals/start-stakeAtom.js @@ -46,8 +46,13 @@ export const startStakeAtom = async ({ const storageNode = await makeStorageNodeChild(chainStorage, VSTORAGE_PATH); const marshaller = await E(board).getPublishingMarshaller(); - const vt = prepareVowTools(makeHeapZone()); - const chainHub = makeChainHub(await agoricNames, vt); + const zone = makeHeapZone(); + const vt = prepareVowTools(zone.subZone('vows')); + const chainHub = makeChainHub( + zone.subZone('chainHub'), + await agoricNames, + vt, + ); const [_, cosmoshub, connectionInfo] = await vt.when( chainHub.getChainsAndConnection('agoric', 'cosmoshub'), diff --git a/packages/orchestration/src/proposals/start-stakeOsmo.js b/packages/orchestration/src/proposals/start-stakeOsmo.js index 84dfadb1337..7ccff4d8658 100644 --- a/packages/orchestration/src/proposals/start-stakeOsmo.js +++ b/packages/orchestration/src/proposals/start-stakeOsmo.js @@ -51,8 +51,13 @@ export const startStakeOsmo = async ({ const storageNode = await makeStorageNodeChild(chainStorage, VSTORAGE_PATH); const marshaller = await E(board).getPublishingMarshaller(); - const vt = prepareVowTools(makeHeapZone()); - const chainHub = makeChainHub(await agoricNames, vt); + const zone = makeHeapZone(); + const vt = prepareVowTools(zone.subZone('vows')); + const chainHub = makeChainHub( + zone.subZone('chainHub'), + await agoricNames, + vt, + ); const [_, osmosis, connectionInfo] = await vt.when( chainHub.getChainsAndConnection('agoric', 'osmosis'), diff --git a/packages/orchestration/src/utils/start-helper.js b/packages/orchestration/src/utils/start-helper.js index f11da8db32c..6b003406bcf 100644 --- a/packages/orchestration/src/utils/start-helper.js +++ b/packages/orchestration/src/utils/start-helper.js @@ -61,6 +61,8 @@ export const provideOrchestration = ( asyncFlow: zone.subZone('asyncFlow'), /** system names for orchestration implementation */ orchestration: zone.subZone('orchestration'), + /** system names for chainHub */ + chainHub: zone.subZone('chainHub'), /** system names for vows */ vows: zone.subZone('vows'), /** system names for zoe */ @@ -74,7 +76,7 @@ export const provideOrchestration = ( const vowTools = prepareVowTools(zones.vows); - const chainHub = makeChainHub(agoricNames, vowTools); + const chainHub = makeChainHub(zones.chainHub, agoricNames, vowTools); const zoeTools = makeZoeTools(zones.zoe, { zcf, vowTools }); diff --git a/packages/orchestration/src/utils/zoe-tools.js b/packages/orchestration/src/utils/zoe-tools.js index abf7f5bb683..dd7fd399c90 100644 --- a/packages/orchestration/src/utils/zoe-tools.js +++ b/packages/orchestration/src/utils/zoe-tools.js @@ -42,6 +42,7 @@ export const makeZoeTools = (zone, { zcf, vowTools }) => { /** * @type {LocalTransfer} */ + // @ts-expect-error XXX typedefs async (srcSeat, localAccount, give) => { !srcSeat.hasExited() || Fail`The seat cannot have exited.`; const { zcfSeat: tempSeat, userSeat: userSeatP } = zcf.makeEmptySeatKit(); diff --git a/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.md b/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.md index 506844dfddd..874ee76b50c 100644 --- a/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.md +++ b/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.md @@ -24,6 +24,17 @@ Generated by [AVA](https://avajs.dev). flowForOutcomeVow: {}, unwrapMap: 'Alleged: weakMapStore', }, + chainHub: { + ChainHub_kindHandle: 'Alleged: kind', + ChainHub_singleton: 'Alleged: ChainHub', + brandDenom: {}, + chainInfos: {}, + connectionInfos: {}, + denom: {}, + lookupChainInfo_kindHandle: 'Alleged: kind', + lookupChainsAndConnection_kindHandle: 'Alleged: kind', + lookupConnectionInfo_kindHandle: 'Alleged: kind', + }, contract: { 'ChainHub Admin_kindHandle': 'Alleged: kind', 'ChainHub Admin_singleton': 'Alleged: ChainHub Admin', @@ -38,6 +49,8 @@ Generated by [AVA](https://avajs.dev). contractState_singleton: 'Alleged: contractState', localTransfer_kindHandle: 'Alleged: kind', localTransfer_singleton: 'Alleged: localTransfer', + log_kindHandle: 'Alleged: kind', + log_singleton: 'Alleged: log', }, }, }, @@ -60,9 +73,14 @@ Generated by [AVA](https://avajs.dev). }, }, vows: { + AdminRetriableFlow_kindHandle: 'Alleged: kind', + AdminRetriableFlow_singleton: 'Alleged: AdminRetriableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', + retriableFlowForOutcomeVow: {}, + }, + zoe: { + localTransfer_kindHandle: 'Alleged: kind', }, - zoe: {}, } diff --git a/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.snap b/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.snap index 1017ab57dbc..e29cb8eb0e2 100644 Binary files a/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.snap and b/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.snap differ diff --git a/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.md b/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.md index 8fa76401b61..f58f00f86e4 100644 --- a/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.md +++ b/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.md @@ -24,6 +24,88 @@ Generated by [AVA](https://avajs.dev). flowForOutcomeVow: {}, unwrapMap: 'Alleged: weakMapStore', }, + chainHub: { + ChainHub_kindHandle: 'Alleged: kind', + ChainHub_singleton: 'Alleged: ChainHub', + brandDenom: { + 'Alleged: BLD brand': 'ubld', + }, + chainInfos: { + agoric: { + chainId: 'agoric-3', + icqEnabled: false, + stakingTokens: [ + { + denom: 'ubld', + }, + ], + }, + cosmoshub: { + chainId: 'cosmoshub-4', + icqEnabled: false, + stakingTokens: [ + { + denom: 'uatom', + }, + ], + }, + }, + connectionInfos: { + 'agoric-3_cosmoshub-4': { + client_id: '07-tendermint-6', + counterparty: { + client_id: '07-tendermint-927', + connection_id: 'connection-649', + prefix: { + key_prefix: 'FIXME', + }, + }, + id: 'connection-8', + state: 3, + transferChannel: { + channelId: 'channel-5', + counterPartyChannelId: 'channel-405', + counterPartyPortId: 'transfer', + ordering: 0, + portId: 'transfer', + state: 3, + version: 'ics20-1', + }, + }, + 'cosmoshub-4_osmosis-1': { + client_id: '07-tendermint-259', + counterparty: { + client_id: '07-tendermint-1', + connection_id: 'connection-1', + prefix: { + key_prefix: 'FIXME', + }, + }, + id: 'connection-257', + state: 3, + transferChannel: { + channelId: 'channel-141', + counterPartyChannelId: 'channel-0', + counterPartyPortId: 'transfer', + ordering: 0, + portId: 'transfer', + state: 3, + version: 'ics20-1', + }, + }, + }, + denom: { + ubld: { + baseDenom: 'ubld', + baseName: 'agoric', + brand: Object @Alleged: BLD brand {}, + chainName: 'agoric', + }, + }, + lookupChainInfo_kindHandle: 'Alleged: kind', + lookupChainsAndConnection_kindHandle: 'Alleged: kind', + lookupConnectionInfo_kindHandle: 'Alleged: kind', + }, contract: { 'ChainHub Admin_kindHandle': 'Alleged: kind', 'ChainHub Admin_singleton': 'Alleged: ChainHub Admin', @@ -129,9 +211,14 @@ Generated by [AVA](https://avajs.dev). }, }, vows: { + AdminRetriableFlow_kindHandle: 'Alleged: kind', + AdminRetriableFlow_singleton: 'Alleged: AdminRetriableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', + retriableFlowForOutcomeVow: {}, + }, + zoe: { + localTransfer_kindHandle: 'Alleged: kind', }, - zoe: {}, } diff --git a/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.snap b/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.snap index 0419690848f..a511de873e1 100644 Binary files a/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.snap and b/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.snap differ diff --git a/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.md b/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.md index dc4bf4aa5a3..7533e219092 100644 --- a/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.md +++ b/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.md @@ -24,6 +24,87 @@ Generated by [AVA](https://avajs.dev). flowForOutcomeVow: {}, unwrapMap: 'Alleged: weakMapStore', }, + chainHub: { + ChainHub_kindHandle: 'Alleged: kind', + ChainHub_singleton: 'Alleged: ChainHub', + brandDenom: {}, + chainInfos: { + agoric: { + chainId: 'agoric-3', + icqEnabled: false, + stakingTokens: [ + { + denom: 'ubld', + }, + ], + }, + osmosis: { + chainId: 'osmosis-1', + icqEnabled: true, + stakingTokens: [ + { + denom: 'uosmo', + }, + ], + }, + stride: { + chainId: 'stride-1', + icqEnabled: false, + stakingTokens: [ + { + denom: 'ustrd', + }, + ], + }, + }, + connectionInfos: { + 'agoric-3_osmosis-1': { + client_id: '07-tendermint-1', + counterparty: { + client_id: '07-tendermint-2109', + connection_id: 'connection-1649', + prefix: { + key_prefix: 'FIXME', + }, + }, + id: 'connection-1', + state: 3, + transferChannel: { + channelId: 'channel-1', + counterPartyChannelId: 'channel-320', + counterPartyPortId: 'transfer', + ordering: 0, + portId: 'transfer', + state: 3, + version: 'ics20-1', + }, + }, + 'agoric-3_stride-1': { + client_id: '07-tendermint-74', + counterparty: { + client_id: '07-tendermint-129', + connection_id: 'connection-118', + prefix: { + key_prefix: 'FIXME', + }, + }, + id: 'connection-68', + state: 3, + transferChannel: { + channelId: 'channel-59', + counterPartyChannelId: 'channel-148', + counterPartyPortId: 'transfer', + ordering: 0, + portId: 'transfer', + state: 3, + version: 'ics20-1', + }, + }, + }, + lookupChainInfo_kindHandle: 'Alleged: kind', + lookupChainsAndConnection_kindHandle: 'Alleged: kind', + lookupConnectionInfo_kindHandle: 'Alleged: kind', + }, contract: { orchestration: { unbondAndLiquidStake: { @@ -53,9 +134,14 @@ Generated by [AVA](https://avajs.dev). }, }, vows: { + AdminRetriableFlow_kindHandle: 'Alleged: kind', + AdminRetriableFlow_singleton: 'Alleged: AdminRetriableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', + retriableFlowForOutcomeVow: {}, + }, + zoe: { + localTransfer_kindHandle: 'Alleged: kind', }, - zoe: {}, } diff --git a/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.snap b/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.snap index 26978e4b049..4e6f357856f 100644 Binary files a/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.snap and b/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.snap differ diff --git a/packages/orchestration/test/exos/chain-hub.test.ts b/packages/orchestration/test/exos/chain-hub.test.ts index bfd7fcb1dd7..aaef51a8b92 100644 --- a/packages/orchestration/test/exos/chain-hub.test.ts +++ b/packages/orchestration/test/exos/chain-hub.test.ts @@ -45,9 +45,9 @@ const connection = { // fresh state for each test const setup = () => { const zone = provideDurableZone('root'); - const vt = prepareSwingsetVowTools(zone); + const vt = prepareSwingsetVowTools(zone.subZone('vows')); const { nameHub, nameAdmin } = makeNameHubKit(); - const chainHub = makeChainHub(nameHub, vt); + const chainHub = makeChainHub(zone.subZone('chainHub'), nameHub, vt); return { chainHub, nameAdmin, vt }; }; diff --git a/packages/orchestration/test/staking-ops.test.ts b/packages/orchestration/test/staking-ops.test.ts index d70287b1874..fb659dd5ed2 100644 --- a/packages/orchestration/test/staking-ops.test.ts +++ b/packages/orchestration/test/staking-ops.test.ts @@ -256,7 +256,7 @@ test('makeAccount() writes to storage', async t => { zone, } = s; const make = prepareCosmosOrchestrationAccountKit(zone, { - chainHub: makeChainHub(agoricNames, vowTools), + chainHub: makeChainHub(zone.subZone('chainHub'), agoricNames, vowTools), makeRecorderKit, timerService: timer, vowTools, @@ -304,7 +304,7 @@ test('withdrawRewards() on StakingAccountHolder formats message correctly', asyn zone, } = s; const make = prepareCosmosOrchestrationAccountKit(zone, { - chainHub: makeChainHub(agoricNames, vowTools), + chainHub: makeChainHub(zone.subZone('chainHub'), agoricNames, vowTools), makeRecorderKit, timerService: timer, vowTools, @@ -349,7 +349,7 @@ test(`delegate; redelegate using invitationMakers`, async t => { zone, } = s; const makeAccountKit = prepareCosmosOrchestrationAccountKit(zone, { - chainHub: makeChainHub(agoricNames, vowTools), + chainHub: makeChainHub(zone.subZone('chainHub'), agoricNames, vowTools), makeRecorderKit, timerService: timer, vowTools, @@ -439,7 +439,7 @@ test(`withdraw rewards using invitationMakers`, async t => { zone, } = s; const makeAccountKit = prepareCosmosOrchestrationAccountKit(zone, { - chainHub: makeChainHub(agoricNames, vowTools), + chainHub: makeChainHub(zone.subZone('chainHub'), agoricNames, vowTools), makeRecorderKit, timerService: timer, vowTools, @@ -487,7 +487,7 @@ test(`undelegate waits for unbonding period`, async t => { zone, } = s; const makeAccountKit = prepareCosmosOrchestrationAccountKit(zone, { - chainHub: makeChainHub(agoricNames, vowTools), + chainHub: makeChainHub(zone.subZone('chainHub'), agoricNames, vowTools), makeRecorderKit, timerService: timer, vowTools, diff --git a/packages/orchestration/test/supports.ts b/packages/orchestration/test/supports.ts index e5832ebe2a5..da33a5cddfa 100644 --- a/packages/orchestration/test/supports.ts +++ b/packages/orchestration/test/supports.ts @@ -149,7 +149,11 @@ export const commonSetup = async (t: ExecutionContext) => { await eventLoopIteration(); }; - const chainHub = makeChainHub(agoricNames, vowTools); + const chainHub = makeChainHub( + rootZone.subZone('chainHub'), + agoricNames, + vowTools, + ); /** * Register BLD if it's not already registered. diff --git a/packages/vow/src/retriable.js b/packages/vow/src/retriable.js new file mode 100644 index 00000000000..b49f2109784 --- /dev/null +++ b/packages/vow/src/retriable.js @@ -0,0 +1,218 @@ +import { Fail } from '@endo/errors'; +import { M } from '@endo/patterns'; +import { PromiseWatcherI } from '@agoric/base-zone'; +import { toPassableCap, VowShape } from './vow-utils.js'; + +/** + * @import {WeakMapStore} from '@agoric/store' + * @import {Zone} from '@agoric/base-zone' + * @import {Vow, VowKit, IsRetryableReason} from './types.js' + * @import {Passable} from '@endo/pass-style' + */ + +/** + * @typedef {object} PreparationOptions + * @property {() => VowKit} makeVowKit + * @property {IsRetryableReason} isRetryableReason + */ + +/** + * @template {Passable[]} [TArgs=any[]] + * @template {any} [TRet=any] + * @typedef {(...args: TArgs) => Promise} RetriableFunc + */ + +const { defineProperties } = Object; + +const RetriableFlowIKit = harden({ + flow: M.interface('Flow', { + restart: M.call().returns(), + getOutcome: M.call().returns(VowShape), + }), + resultWatcher: PromiseWatcherI, +}); + +const AdminRetriableFlowI = M.interface('RetriableFlowAdmin', { + getFlowForOutcomeVow: M.call(VowShape).returns(M.opt(M.remotable('flow'))), +}); + +/** + * @param {Zone} outerZone + * @param {PreparationOptions} outerOptions + */ +export const prepareRetriableTools = (outerZone, outerOptions) => { + const { makeVowKit, isRetryableReason } = outerOptions; + + /** + * So we can give out wrapper functions easily and recover flow objects + * for their activations later. + */ + const flowForOutcomeVowKey = outerZone.mapStore( + 'retriableFlowForOutcomeVow', + { + keyShape: M.remotable('toPassableCap'), + valueShape: M.remotable('flow'), // isDone === false + }, + ); + + /** + * @param {Zone} zone + * @param {string} tag + * @param {RetriableFunc} retriableFunc + */ + const prepareRetriableFlowKit = (zone, tag, retriableFunc) => { + typeof retriableFunc === 'function' || + Fail`retriableFunc must be a callable function ${retriableFunc}`; + + const internalMakeRetriableFlowKit = zone.exoClassKit( + tag, + RetriableFlowIKit, + activationArgs => { + harden(activationArgs); + + return { + activationArgs, // restarting the retriable function uses the original args + outcomeKit: makeVowKit(), // outcome of activation as vow + lastRetryReason: undefined, + runs: 0n, + isDone: false, // persistently done + }; + }, + { + flow: { + /** + * Calls the retriable function, either for the initial run or when + * the result of the previous run fails with a retriable reason. + */ + restart() { + const { state, facets } = this; + const { activationArgs, isDone } = state; + const { flow, resultWatcher } = facets; + + !isDone || + // separate line so I can set a breakpoint + Fail`Cannot restart a done retriable flow ${flow}`; + + const runId = state.runs + 1n; + state.runs = runId; + + let resultP; + try { + resultP = Promise.resolve(retriableFunc(...activationArgs)); + } catch (err) { + resultP = Promise.resolve(() => Promise.reject(err)); + } + + outerZone.watchPromise(harden(resultP), resultWatcher, runId); + }, + getOutcome() { + const { state } = this; + const { outcomeKit } = state; + return outcomeKit.vow; + }, + }, + resultWatcher: { + onFulfilled(value, runId) { + const { state } = this; + const { runs, outcomeKit } = state; + if (runId !== runs) return; + !state.isDone || + Fail`Cannot resolve a done retriable flow ${this.facets.flow}`; + outcomeKit.resolver.resolve(value); + flowForOutcomeVowKey.delete(toPassableCap(outcomeKit.vow)); + state.isDone = true; + }, + onRejected(reason, runId) { + const { state } = this; + const { runs, outcomeKit } = state; + if (runId !== runs) return; + !state.isDone || + Fail`Cannot reject a done retriable flow ${this.facets.flow}`; + const retryReason = isRetryableReason( + reason, + state.lastRetryReason, + ); + if (retryReason) { + state.lastRetryReason = retryReason; + this.facets.flow.restart(); + } else { + outcomeKit.resolver.reject(reason); + flowForOutcomeVowKey.delete(toPassableCap(outcomeKit.vow)); + state.isDone = true; + } + }, + }, + }, + ); + const makeRetriableFlowKit = activationArgs => { + const retriableKit = internalMakeRetriableFlowKit(activationArgs); + const { flow } = retriableKit; + + const vow = flow.getOutcome(); + flowForOutcomeVowKey.init(toPassableCap(vow), flow); + flow.restart(); + return retriableKit; + }; + return harden(makeRetriableFlowKit); + }; + + /** + * @template {RetriableFunc} F + * @param {Zone} zone + * @param {string} tag + * @param {F} retriableFunc + */ + const retriable = (zone, tag, retriableFunc) => { + const makeRetriableKit = prepareRetriableFlowKit(zone, tag, retriableFunc); + const wrapperFuncName = `${tag}_retriable`; + + const wrapperFunc = { + /** @type {(...args: Parameters) => Vow>>} */ + [wrapperFuncName](...args) { + const { flow } = makeRetriableKit(args); + return flow.getOutcome(); + }, + }[wrapperFuncName]; + defineProperties(wrapperFunc, { + length: { value: retriableFunc.length }, + }); + return harden(wrapperFunc); + }; + + const adminRetriableFlow = outerZone.exo( + 'AdminRetriableFlow', + AdminRetriableFlowI, + { + getFlowForOutcomeVow(outcomeVow) { + return flowForOutcomeVowKey.get(toPassableCap(outcomeVow)); + }, + }, + ); + + return harden({ + prepareRetriableFlowKit, + adminRetriableFlow, + retriable, + }); +}; +harden(prepareRetriableTools); + +/** + * @typedef {ReturnType} RetriableTools + */ + +/** + * @typedef {RetriableTools['adminRetriableFlow']} AdminRetriableFlow + */ + +/** + * @typedef {ReturnType} MakeRetriableFlowKit + */ + +/** + * @typedef {ReturnType} RetriableFlowKit + */ + +/** + * @typedef {RetriableFlowKit['flow']} RetriableFlow + */ diff --git a/packages/vow/src/tools.js b/packages/vow/src/tools.js index ff35539726c..432d2ffbb83 100644 --- a/packages/vow/src/tools.js +++ b/packages/vow/src/tools.js @@ -3,6 +3,7 @@ import { makeAsVow } from './vow-utils.js'; import { prepareVowKit } from './vow.js'; import { prepareWatchUtils } from './watch-utils.js'; import { prepareWatch } from './watch.js'; +import { prepareRetriableTools } from './retriable.js'; import { makeWhen } from './when.js'; /** @@ -33,23 +34,10 @@ export const prepareBasicVowTools = (zone, powers = {}) => { const watchUtils = makeWatchUtils(); const asVow = makeAsVow(makeVowKit); - /** - * TODO FIXME make this real - * Create a function that retries the given function if the underlying - * functions rejects due to upgrade disconnection. - * - * @template {(...args: any[]) => Promise} F - * @param {Zone} fnZone - the zone for the named function - * @param {string} name - * @param {F} fn - * @returns {F extends (...args: infer Args) => Promise ? (...args: Args) => Vow : never} - */ - const retriable = - (fnZone, name, fn) => - // @ts-expect-error cast - (...args) => { - return watch(fn(...args)); - }; + const { retriable } = prepareRetriableTools(zone, { + makeVowKit, + isRetryableReason, + }); /** * Vow-tolerant implementation of Promise.all. diff --git a/packages/vow/vat.js b/packages/vow/vat.js index a486730b9c4..ad7d177d409 100644 --- a/packages/vow/vat.js +++ b/packages/vow/vat.js @@ -5,7 +5,10 @@ /* global globalThis */ // @ts-check -import { isUpgradeDisconnection } from '@agoric/internal/src/upgrade-api.js'; +import { + isUpgradeDisconnection, + isAbandonedError, +} from '@agoric/internal/src/upgrade-api.js'; import { makeHeapZone } from '@agoric/base-zone/heap.js'; import { prepareBasicVowTools } from './src/tools.js'; @@ -15,11 +18,16 @@ import makeE from './src/E.js'; const isRetryableReason = (reason, priorRetryValue) => { if ( isUpgradeDisconnection(reason) && - (!priorRetryValue || + (!isUpgradeDisconnection(priorRetryValue) || reason.incarnationNumber > priorRetryValue.incarnationNumber) ) { return reason; } + // For abandoned errors there is no way to differentiate errors from + // consecutive upgrades + if (isAbandonedError(reason) && !isAbandonedError(priorRetryValue)) { + return reason; + } return undefined; }; diff --git a/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.md b/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.md index 0d19048e384..40361190c32 100644 --- a/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.md +++ b/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.md @@ -9,11 +9,14 @@ Generated by [AVA](https://avajs.dev). > contract baggage after start { + AdminRetriableFlow_kindHandle: 'Alleged: kind', + AdminRetriableFlow_singleton: 'Alleged: AdminRetriableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', publicFacet_kindHandle: 'Alleged: kind', publicFacet_singleton: 'Alleged: publicFacet', + retriableFlowForOutcomeVow: {}, vowResolver: { resolver: Object @Alleged: VowInternalsKit resolver {}, vow: Object @Vow { diff --git a/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.snap b/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.snap index 756b545154f..171dc6777c1 100644 Binary files a/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.snap and b/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.snap differ