From b9f677cdf3a33d71dd3368487371812ce4304a57 Mon Sep 17 00:00:00 2001
From: Turadg Aleahmad <turadg@agoric.com>
Date: Thu, 11 May 2023 07:26:05 -0700
Subject: [PATCH] test(bootstrap): restart all contracts

---
 .../src/proposals/core-proposal.js            |   1 +
 .../src/proposals/econ-behaviors.js           |  25 +-
 .../src/proposals/startEconCommittee.js       |  20 +-
 .../inter-protocol/src/proposals/startPSM.js  |   1 +
 packages/vats/package.json                    |   1 +
 packages/vats/scripts/restart-vats.js         |  23 ++
 .../src/proposals/restart-vats-proposal.js    | 133 +++++++++
 .../test/bootstrapTests/test-vats-restart.js  | 258 ++++++++++++++++++
 .../bootstrapTests/test-vaults-upgrade.js     |   1 +
 9 files changed, 448 insertions(+), 15 deletions(-)
 create mode 100644 packages/vats/scripts/restart-vats.js
 create mode 100644 packages/vats/src/proposals/restart-vats-proposal.js
 create mode 100644 packages/vats/test/bootstrapTests/test-vats-restart.js

diff --git a/packages/inter-protocol/src/proposals/core-proposal.js b/packages/inter-protocol/src/proposals/core-proposal.js
index c949b7626b4..017b9b6bf2f 100644
--- a/packages/inter-protocol/src/proposals/core-proposal.js
+++ b/packages/inter-protocol/src/proposals/core-proposal.js
@@ -100,6 +100,7 @@ const SHARED_MAIN_MANIFEST = harden({
     consume: {
       board: 'board',
       chainStorage: true,
+      diagnostics: true,
       feeMintAccess: 'zoe',
       chainTimerService: 'timer',
       zoe: 'zoe',
diff --git a/packages/inter-protocol/src/proposals/econ-behaviors.js b/packages/inter-protocol/src/proposals/econ-behaviors.js
index 87276d92978..0bdbd6926b5 100644
--- a/packages/inter-protocol/src/proposals/econ-behaviors.js
+++ b/packages/inter-protocol/src/proposals/econ-behaviors.js
@@ -76,6 +76,7 @@ export const setupReserve = async ({
     feeMintAccess: feeMintAccessP,
     chainStorage,
     chainTimerService,
+    diagnostics,
     zoe,
     economicCommitteeCreatorFacet: committeeCreator,
   },
@@ -120,19 +121,20 @@ export const setupReserve = async ({
       },
     }),
   );
+  const privateArgs = {
+    governed: {
+      feeMintAccess,
+      initialPoserInvitation: poserInvitation,
+      marshaller,
+      storageNode,
+    },
+  };
   /** @type {GovernorStartedInstallationKit<typeof reserveInstallation>} */
   const g = await E(zoe).startInstance(
     governorInstallation,
     {},
     reserveGovernorTerms,
-    {
-      governed: {
-        feeMintAccess,
-        initialPoserInvitation: poserInvitation,
-        marshaller,
-        storageNode,
-      },
-    },
+    privateArgs,
     'reserve.governor',
   );
 
@@ -155,6 +157,9 @@ export const setupReserve = async ({
       governorAdminFacet: g.adminFacet,
     }),
   );
+  const { instancePrivateArgs } = await diagnostics;
+  instancePrivateArgs.set(instance, privateArgs.governed);
+  instancePrivateArgs.set(g.instance, privateArgs);
 
   reserveInstanceProducer.resolve(instance);
   reserveGovernor.resolve(g.instance);
@@ -454,7 +459,9 @@ export const startRewardDistributor = async ({
     ),
   });
 
-  feeDistributorKit.resolve(instanceKit);
+  feeDistributorKit.resolve(
+    harden({ ...instanceKit, label: 'feeDistributor' }),
+  );
   feeDistributorP.resolve(instanceKit.instance);
 
   // Initialize the periodic collectors list if we don't have one.
diff --git a/packages/inter-protocol/src/proposals/startEconCommittee.js b/packages/inter-protocol/src/proposals/startEconCommittee.js
index e684109f783..ef658cebd09 100644
--- a/packages/inter-protocol/src/proposals/startEconCommittee.js
+++ b/packages/inter-protocol/src/proposals/startEconCommittee.js
@@ -28,7 +28,7 @@ const sanitizePathSegment = name => {
  */
 export const startEconomicCommittee = async (
   {
-    consume: { board, chainStorage, zoe },
+    consume: { board, chainStorage, diagnostics, zoe },
     produce: { economicCommitteeKit, economicCommitteeCreatorFacet },
     installation: {
       consume: { committee },
@@ -59,19 +59,26 @@ export const startEconomicCommittee = async (
   // NB: committee must only publish what it intended to be public
   const marshaller = await E(board).getPublishingMarshaller();
 
+  const privateArgs = {
+    storageNode,
+    marshaller,
+  };
   const startResult = await E(zoe).startInstance(
     committee,
     {},
     { committeeName, committeeSize, ...rest },
-    {
-      storageNode,
-      marshaller,
-    },
+    privateArgs,
     'economicCommittee',
   );
   const { creatorFacet, instance } = startResult;
 
-  economicCommitteeKit.resolve(startResult);
+  economicCommitteeKit.resolve(
+    // XXX should startInstance return its label?
+    harden({ ...startResult, label: 'economicCommittee' }),
+  );
+  const { instancePrivateArgs } = await diagnostics;
+  instancePrivateArgs.set(startResult.instance, privateArgs);
+
   economicCommitteeCreatorFacet.resolve(creatorFacet);
   economicCommittee.resolve(instance);
 };
@@ -82,6 +89,7 @@ export const ECON_COMMITTEE_MANIFEST = harden({
     consume: {
       board: true,
       chainStorage: true,
+      diagnostics: true,
       zoe: true,
     },
     produce: {
diff --git a/packages/inter-protocol/src/proposals/startPSM.js b/packages/inter-protocol/src/proposals/startPSM.js
index 6d4b02e50d9..034fad7e341 100644
--- a/packages/inter-protocol/src/proposals/startPSM.js
+++ b/packages/inter-protocol/src/proposals/startPSM.js
@@ -259,6 +259,7 @@ export const startPSM = async (
   /** @type {MapStore<Brand,psmKit>} */
   const psmKitMap = await psmKit;
 
+  // TODO init into governedContractKits too to simplify testing
   psmKitMap.init(anchorBrand, newpsmKit);
   const instanceAdmin = E(agoricNamesAdmin).lookupAdmin('instance');
 
diff --git a/packages/vats/package.json b/packages/vats/package.json
index 88cd91fe4a9..0b919d89bc3 100644
--- a/packages/vats/package.json
+++ b/packages/vats/package.json
@@ -13,6 +13,7 @@
     "build:boot-viz-gov": "node src/authorityViz.js --gov>docs/boot-gov.dot && dot -Tsvg docs/boot-gov.dot >docs/boot-gov.dot.svg",
     "build:boot-viz-sim": "node src/authorityViz.js --sim-chain >docs/boot-sim.dot && dot -Tsvg docs/boot-sim.dot >docs/boot-sim.dot.svg",
     "build:boot-viz-sim-gov": "node src/authorityViz.js --sim-chain --gov >docs/boot-sim-gov.dot && dot -Tsvg docs/boot-sim-gov.dot >docs/boot-sim-gov.dot.svg",
+    "build:restart-vats-proposal": "agoric run scripts/restart-vats.js",
     "prepack": "tsc --build jsconfig.build.json",
     "postpack": "git clean -f '*.d.ts*'",
     "test": "ava",
diff --git a/packages/vats/scripts/restart-vats.js b/packages/vats/scripts/restart-vats.js
new file mode 100644
index 00000000000..0e46b92ca47
--- /dev/null
+++ b/packages/vats/scripts/restart-vats.js
@@ -0,0 +1,23 @@
+import { makeHelpers } from '@agoric/deploy-script-support';
+
+/** @type {import('@agoric/deploy-script-support/src/externalTypes.js').ProposalBuilder} */
+export const defaultProposalBuilder = async () => {
+  // An includelist isn't necessary because the collections are known to be complete (tested in test-vaults-upgrade.js)
+  const skip = [
+    // can be replaced instead of upgraded
+    'auctioneer',
+    'feeDistributor',
+    // skip so vaultManager can have prices upon restart; these have been tested as restartable
+    'scaledPriceAuthority-ATOM',
+  ];
+
+  return harden({
+    sourceSpec: '../src/proposals/restart-vats-proposal.js',
+    getManifestCall: ['getManifestForRestart', harden({ skip })],
+  });
+};
+
+export default async (homeP, endowments) => {
+  const { writeCoreProposal } = await makeHelpers(homeP, endowments);
+  await writeCoreProposal('restart-vats', defaultProposalBuilder);
+};
diff --git a/packages/vats/src/proposals/restart-vats-proposal.js b/packages/vats/src/proposals/restart-vats-proposal.js
new file mode 100644
index 00000000000..5ea51b36f94
--- /dev/null
+++ b/packages/vats/src/proposals/restart-vats-proposal.js
@@ -0,0 +1,133 @@
+/* eslint-disable no-await-in-loop */
+// @ts-check
+
+import { Fail } from '@agoric/assert';
+import { deeplyFulfilledObject, makeTracer } from '@agoric/internal';
+import { M, mustMatch } from '@agoric/store';
+import { E, getInterfaceOf } from '@endo/far';
+
+const trace = makeTracer('RV');
+
+const HR = '----------------';
+
+/**
+ * TODO cover each of these collections
+ * - [x] contractKits
+ * - [ ] psmKit
+ * - [x] governedContractKits
+ * - [ ] vatStore
+ */
+
+/**
+ * @param {BootstrapPowers & import('@agoric/inter-protocol/src/proposals/econ-behaviors.js').EconomyBootstrapSpace} space
+ * @param {object} config
+ * @param {{ skip: Array<string>}} config.options
+ */
+export const restartVats = async ({ consume }, { options }) => {
+  console.log(HR);
+  console.log(HR);
+  trace('restartVats start', options);
+  mustMatch(options, harden({ skip: M.arrayOf(M.string()) }));
+
+  trace('awaiting VaultFactorKit as a proxy for "bootstrap done"');
+  await consume.vaultFactoryKit;
+
+  trace('testing restarts');
+  const { contractKits, governedContractKits } = await deeplyFulfilledObject(
+    harden({
+      contractKits: consume.contractKits,
+      governedContractKits: consume.governedContractKits,
+    }),
+  );
+
+  const { instancePrivateArgs } = await consume.diagnostics;
+
+  const failures = [];
+  /**
+   *
+   * @param {string} debugName
+   * @param {Instance} instance
+   * @param {ERef<AdminFacet>} adminFacet
+   */
+  const tryRestart = async (debugName, instance, adminFacet) => {
+    // TODO document that privateArgs cannot contain promises
+    // TODO try making all the contract starts take resolved values
+    const privateArgs = await deeplyFulfilledObject(
+      // @ts-expect-error cast
+      harden(instancePrivateArgs.get(instance) || {}),
+    );
+
+    console.log(HR);
+    console.log(HR);
+    console.log(HR);
+    console.log(HR);
+    trace('tryRestart', debugName, privateArgs);
+
+    if (options.skip.includes(debugName)) {
+      trace('SKIPPED', debugName);
+      return;
+    }
+    try {
+      await E(adminFacet).restartContract(privateArgs);
+      trace('RESTARTED', debugName);
+    } catch (err) {
+      trace('🚨 RESTART FAILED', debugName, err);
+      failures.push(debugName);
+    }
+  };
+
+  // iterate over the two contractKits and use the adminFacet to restartContract
+  for (const kit of contractKits.values()) {
+    const debugName =
+      kit.label || getInterfaceOf(kit.publicFacet) || 'UNLABELED';
+    if (debugName !== kit.label) {
+      console.warn('MISSING LABEL:', kit);
+    }
+    await tryRestart(debugName, kit.instance, kit.adminFacet);
+  }
+
+  for (const kit of governedContractKits.values()) {
+    const debugName =
+      kit.label || getInterfaceOf(kit.publicFacet) || 'UNLABELED';
+    if (debugName !== kit.label) {
+      console.warn('MISSING LABEL:', kit);
+    }
+
+    trace('restarting governed', debugName);
+    await tryRestart(debugName, kit.instance, kit.adminFacet);
+
+    trace('restarting governor of', debugName);
+    await tryRestart(
+      `${debugName} [Governor]`,
+      kit.governor,
+      kit.governorAdminFacet,
+    );
+  }
+
+  trace('restartVats done with ', failures.length, 'failures');
+  console.log(HR);
+  if (failures.length) {
+    Fail`restart failed for ${failures.join(',')}`;
+  }
+  console.log(HR);
+};
+harden(restartVats);
+
+export const getManifestForRestart = (_powers, options) => ({
+  manifest: {
+    [restartVats.name]: {
+      consume: {
+        contractKits: true,
+        diagnostics: true,
+        governedContractKits: true,
+        loadCriticalVat: true,
+        zoe: 'zoe',
+        provisioning: 'provisioning',
+        vaultFactoryKit: true,
+      },
+      produce: {},
+    },
+  },
+  options,
+});
+harden(getManifestForRestart);
diff --git a/packages/vats/test/bootstrapTests/test-vats-restart.js b/packages/vats/test/bootstrapTests/test-vats-restart.js
new file mode 100644
index 00000000000..1468f01475c
--- /dev/null
+++ b/packages/vats/test/bootstrapTests/test-vats-restart.js
@@ -0,0 +1,258 @@
+// @ts-check
+/* global process */
+/**
+ * @file Bootstrap test of restarting (almost) all vats
+ */
+import { test as anyTest } from '@agoric/zoe/tools/prepare-test-env-ava.js';
+
+import * as processAmbient from 'child_process';
+import * as fsAmbient from 'fs';
+import { Fail } from '@agoric/assert';
+import { eventLoopIteration } from '@agoric/internal/src/testing-utils.js';
+import { Offers } from '@agoric/inter-protocol/src/clientSupport.js';
+import { makeAgoricNamesRemotesFromFakeStorage } from '../../tools/board-utils.js';
+import { makeSwingsetTestKit, makeWalletFactoryDriver } from './supports.js';
+
+/**
+ * @type {import('ava').TestFn<Awaited<ReturnType<typeof makeTestContext>>>}
+ */
+const test = anyTest;
+
+const PLATFORM_CONFIG = '@agoric/vats/decentral-devnet-config.json';
+
+// presently all these tests use one collateral manager
+const collateralBrandKey = 'ATOM';
+
+/**
+ * @param {object} powers
+ * @param {Pick<typeof import('node:child_process'), 'execFileSync' >} powers.childProcess
+ * @param {typeof import('node:fs/promises')} powers.fs
+ */
+const makeProposalExtractor = ({ childProcess, fs }) => {
+  const getPkgPath = (pkg, fileName = '') =>
+    new URL(`../../../${pkg}/${fileName}`, import.meta.url).pathname;
+
+  const runPackageScript = async (pkg, name, env) => {
+    console.warn(pkg, 'running package script:', name);
+    const pkgPath = getPkgPath(pkg);
+    return childProcess.execFileSync('yarn', ['run', name], {
+      cwd: pkgPath,
+      env,
+    });
+  };
+
+  const loadJSON = async filePath =>
+    harden(JSON.parse(await fs.readFile(filePath, 'utf8')));
+
+  // XXX parses the output to find the files but could write them to a path that can be traversed
+  /** @param {string} txt */
+  const parseProposalParts = txt => {
+    const evals = [
+      ...txt.matchAll(/swingset-core-eval (?<permit>\S+) (?<script>\S+)/g),
+    ].map(m => {
+      if (!m.groups) throw Fail`Invalid proposal output ${m[0]}`;
+      const { permit, script } = m.groups;
+      return { permit, script };
+    });
+    evals.length ||
+      Fail`No swingset-core-eval found in proposal output: ${txt}`;
+
+    const bundles = [
+      ...txt.matchAll(/swingset install-bundle @([^\n]+)/gm),
+    ].map(([, bundle]) => bundle);
+    bundles.length || Fail`No bundles found in proposal output: ${txt}`;
+
+    return { evals, bundles };
+  };
+
+  /**
+   * @param {object} options
+   * @param {string} options.package
+   * @param {string} options.packageScriptName
+   * @param {Record<string, string>} [options.env]
+   */
+  const buildAndExtract = async ({
+    package: packageName,
+    packageScriptName,
+    env = {},
+  }) => {
+    const scriptEnv = Object.assign(Object.create(process.env), env);
+    // XXX use '@agoric/inter-protocol'?
+    const out = await runPackageScript(
+      packageName,
+      packageScriptName,
+      scriptEnv,
+    );
+    const built = parseProposalParts(out.toString());
+
+    const loadAndRmPkgFile = async fileName => {
+      const filePath = getPkgPath(packageName, fileName);
+      const content = await fs.readFile(filePath, 'utf8');
+      await fs.rm(filePath);
+      return content;
+    };
+
+    const evalsP = Promise.all(
+      built.evals.map(async ({ permit, script }) => {
+        const [permits, code] = await Promise.all([
+          loadAndRmPkgFile(permit),
+          loadAndRmPkgFile(script),
+        ]);
+        return { json_permits: permits, js_code: code };
+      }),
+    );
+
+    const bundlesP = Promise.all(
+      built.bundles.map(
+        async bundleFile =>
+          /** @type {Promise<EndoZipBase64Bundle>} */ (loadJSON(bundleFile)),
+      ),
+    );
+    return Promise.all([evalsP, bundlesP]).then(([evals, bundles]) => ({
+      evals,
+      bundles,
+    }));
+  };
+  return buildAndExtract;
+};
+
+const makeTestContext = async t => {
+  console.time('DefaultTestContext');
+  const swingsetTestKit = await makeSwingsetTestKit(t, 'bundles/vaults', {
+    configSpecifier: PLATFORM_CONFIG,
+  });
+
+  const { runUtils, storage } = swingsetTestKit;
+  console.timeLog('DefaultTestContext', 'swingsetTestKit');
+  const { EV } = runUtils;
+
+  // Wait for ATOM to make it into agoricNames
+  await EV.vat('bootstrap').consumeItem('vaultFactoryKit');
+  console.timeLog('DefaultTestContext', 'vaultFactoryKit');
+
+  await eventLoopIteration();
+
+  // has to be late enough for agoricNames data to have been published
+  const agoricNamesRemotes = makeAgoricNamesRemotesFromFakeStorage(
+    swingsetTestKit.storage,
+  );
+  agoricNamesRemotes.brand.ATOM || Fail`ATOM brand not yet defined`;
+  console.timeLog('DefaultTestContext', 'agoricNamesRemotes');
+
+  const walletFactoryDriver = await makeWalletFactoryDriver(
+    runUtils,
+    storage,
+    agoricNamesRemotes,
+  );
+  console.timeLog('DefaultTestContext', 'walletFactoryDriver');
+
+  console.timeEnd('DefaultTestContext');
+
+  const buildProposal = makeProposalExtractor({
+    childProcess: processAmbient,
+    fs: fsAmbient.promises,
+  });
+
+  return {
+    ...swingsetTestKit,
+    agoricNamesRemotes,
+    walletFactoryDriver,
+    buildProposal,
+  };
+};
+
+test.before(async t => {
+  t.context = await makeTestContext(t);
+});
+test.after.always(t => t.context.shutdown?.());
+
+const walletAddr = 'agoric1a';
+
+test.serial('open vault', async t => {
+  const { walletFactoryDriver } = t.context;
+
+  const wd = await walletFactoryDriver.provideSmartWallet(walletAddr);
+  t.true(wd.isNew);
+
+  await wd.executeOfferMaker(Offers.vaults.OpenVault, {
+    offerId: 'open1',
+    collateralBrandKey,
+    wantMinted: 5.0,
+    giveCollateral: 9.0,
+  });
+
+  t.like(wd.getLatestUpdateRecord(), {
+    updated: 'offerStatus',
+    status: { id: 'open1', numWantsSatisfied: 1 },
+  });
+});
+
+test.serial('run restart-vats proposal', async t => {
+  const { controller, buildProposal } = t.context;
+
+  t.log('building proposal');
+  const proposal = await buildProposal({
+    package: 'vats',
+    packageScriptName: 'build:restart-vats-proposal',
+  });
+
+  for await (const bundle of proposal.bundles) {
+    await controller.validateAndInstallBundle(bundle);
+  }
+  t.log('installed', proposal.bundles.length, 'bundles');
+
+  t.log('launching proposal');
+  const bridgeMessage = {
+    type: 'CORE_EVAL',
+    evals: proposal.evals,
+  };
+  t.log({ bridgeMessage });
+  const { EV } = t.context.runUtils;
+  /** @type {ERef<import('../../src/types.js').BridgeHandler>} */
+  const coreEvalBridgeHandler = await EV.vat('bootstrap').consumeItem(
+    'coreEvalBridgeHandler',
+  );
+  await EV(coreEvalBridgeHandler).fromBridge(bridgeMessage);
+
+  t.log('restart-vats proposal executed');
+  t.pass(); // reached here without throws
+});
+
+test.serial('read metrics', async t => {
+  const { EV } = t.context.runUtils;
+
+  /** @type {Awaited<import('@agoric/inter-protocol/src/proposals/econ-behaviors.js').EconomyBootstrapSpace['consume']['vaultFactoryKit']>} */
+  const vaultFactoryKit = await EV.vat('bootstrap').consumeItem(
+    'vaultFactoryKit',
+  );
+
+  const vfTopics = await EV(vaultFactoryKit.publicFacet).getPublicTopics();
+
+  const vfMetricsPath = await EV.get(vfTopics.metrics).storagePath;
+  t.is(vfMetricsPath, 'published.vaultFactory.metrics');
+
+  await t.throwsAsync(
+    EV(vfTopics.metrics.subscriber).getUpdateSince(),
+    undefined,
+    'reconnecting subscriber not expected to work',
+  );
+});
+
+test.serial('open second vault', async t => {
+  const { walletFactoryDriver } = t.context;
+
+  const wd = await walletFactoryDriver.provideSmartWallet(walletAddr);
+  t.false(wd.isNew);
+
+  await wd.executeOfferMaker(Offers.vaults.OpenVault, {
+    offerId: 'open2',
+    collateralBrandKey,
+    wantMinted: 5.0,
+    giveCollateral: 9.0,
+  });
+
+  t.like(wd.getLatestUpdateRecord(), {
+    updated: 'offerStatus',
+    status: { id: 'open2', numWantsSatisfied: 1 },
+  });
+});
diff --git a/packages/vats/test/bootstrapTests/test-vaults-upgrade.js b/packages/vats/test/bootstrapTests/test-vaults-upgrade.js
index 353b9ad17c8..d19a96bc0ae 100644
--- a/packages/vats/test/bootstrapTests/test-vaults-upgrade.js
+++ b/packages/vats/test/bootstrapTests/test-vaults-upgrade.js
@@ -463,6 +463,7 @@ test.serial(
       EV.vat('bootstrap').snapshotStore(await EV(powerStore).get(name));
 
     const contractKits = await getStoreSnapshot('contractKits');
+    // TODO refactor the entries to go into governedContractKits too (so the latter is sufficient to test)
     const psmKit = await getStoreSnapshot('psmKit');
     const governedContractKits = await getStoreSnapshot('governedContractKits');
     const vatStore = await getStoreSnapshot('vatStore');