test orch flow upgrades by bridge message deferral (#9755)

refs: #9303

## Description

Tests restarting an orchestration flow while an IBC message is pending response. 

#9303 will be closed when this and #9719 have landed.

### Security Considerations
none

### Scaling Considerations
none

### Documentation Considerations
none

### Testing Considerations
per se

### Upgrade Considerations
helps, no change

Author: mergify[bot]

Diff for: packages/boot/test/bootstrapTests/orchestration.test.ts

@@ -136,7 +136,7 @@ test.serial('stakeAtom - smart wallet', async t => {
     buildProposal,
     evalProposal,
     agoricNamesRemotes,
-    flushInboundQueue,
+    bridgeUtils: { flushInboundQueue },
     readLatest,
   } = t.context;
 
@@ -260,7 +260,7 @@ test('basic-flows', async t => {
     evalProposal,
     agoricNamesRemotes,
     readLatest,
-    flushInboundQueue,
+    bridgeUtils: { flushInboundQueue },
   } = t.context;
 
   await evalProposal(
@@ -341,7 +341,7 @@ test.serial('basic-flows - portfolio holder', async t => {
     evalProposal,
     readLatest,
     agoricNamesRemotes,
-    flushInboundQueue,
+    bridgeUtils: { flushInboundQueue },
   } = t.context;
 
   await evalProposal(

Diff for: packages/boot/test/bootstrapTests/vtransfer.test.ts

@@ -24,8 +24,12 @@ test.before(async t => (t.context = await makeDefaultTestContext(t)));
 test.after.always(t => t.context.shutdown?.());
 
 test('vtransfer', async t => {
-  const { buildProposal, evalProposal, getOutboundMessages, runUtils } =
-    t.context;
+  const {
+    buildProposal,
+    evalProposal,
+    bridgeUtils: { getOutboundMessages },
+    runUtils,
+  } = t.context;
   const { EV } = runUtils;
 
   // Pull what transfer-proposal produced into local scope

Diff for: packages/boot/test/orchestration/restart-contracts.test.ts

@@ -24,7 +24,7 @@ test.serial('sendAnywhere', async t => {
     walletFactoryDriver,
     buildProposal,
     evalProposal,
-    flushInboundQueue,
+    bridgeUtils: { flushInboundQueue },
   } = t.context;
 
   const { IST } = t.context.agoricNamesRemotes.brand;
@@ -114,7 +114,7 @@ test('stakeAtom', async t => {
     buildProposal,
     evalProposal,
     agoricNamesRemotes,
-    flushInboundQueue,
+    bridgeUtils: { flushInboundQueue },
     readLatest,
   } = t.context;
 
@@ -167,3 +167,107 @@ test('stakeAtom', async t => {
   t.is(await flushInboundQueue(), 1);
   t.true(hasResult(wd.getLatestUpdateRecord()));
 });
+
+// Tests restart of an orchestration() flow while an IBC response is pending.
+//
+// TODO consider testing this pausing during any pending IBC message. It'll need
+// to fresh contract state on each iteration, and since this is a bootstrap test
+// that means either restarting bootstrap or starting a new contract and
+// restarting that one. For them to share bootstrap they'll each need a unique
+// instance name, which will require paramatizing the the two builders scripts
+// and the two core-eval functions.
+test.serial('basicFlows', async t => {
+  const {
+    walletFactoryDriver,
+    buildProposal,
+    evalProposal,
+    bridgeUtils: { getInboundQueueLength, flushInboundQueue },
+  } = t.context;
+
+  t.log('start basicFlows');
+  await evalProposal(
+    buildProposal('@agoric/builders/scripts/orchestration/init-basic-flows.js'),
+  );
+
+  t.log('making offer');
+  const wallet = await walletFactoryDriver.provideSmartWallet('agoric1test');
+  const id1 = 'make-orch-account';
+  // send because it won't resolve
+  await wallet.sendOffer({
+    id: id1,
+    invitationSpec: {
+      source: 'agoricContract',
+      instancePath: ['basicFlows'],
+      callPipe: [['makeOrchAccountInvitation']],
+    },
+    proposal: {},
+    offerArgs: {
+      chainName: 'cosmoshub',
+    },
+  });
+  // no errors and no result yet
+  t.like(wallet.getLatestUpdateRecord(), {
+    status: {
+      id: id1,
+      error: undefined,
+      numWantsSatisfied: 1,
+      payouts: {},
+      result: undefined, // no property
+    },
+  });
+  t.is(getInboundQueueLength(), 1);
+
+  const id2 = 'makePortfolio';
+  await wallet.sendOffer({
+    id: id2,
+    invitationSpec: {
+      source: 'agoricContract',
+      instancePath: ['basicFlows'],
+      callPipe: [['makePortfolioAccountInvitation']],
+    },
+    proposal: {},
+    offerArgs: {
+      chainNames: ['agoric', 'cosmoshub'],
+    },
+  });
+  // no errors and no result yet
+  t.like(wallet.getLatestUpdateRecord(), {
+    status: {
+      id: id2,
+      error: undefined,
+      numWantsSatisfied: 1,
+      payouts: {},
+      result: undefined, // no property
+    },
+  });
+  t.is(getInboundQueueLength(), 2);
+
+  t.log('restart basicFlows');
+  await evalProposal(
+    buildProposal('@agoric/builders/scripts/testing/restart-basic-flows.js'),
+  );
+
+  t.log('flush and verify results');
+  const beforeFlush = wallet.getLatestUpdateRecord();
+  t.like(beforeFlush, {
+    status: {
+      result: undefined,
+    },
+  });
+  t.is(await flushInboundQueue(1), 1);
+  t.like(wallet.getLatestUpdateRecord(), {
+    status: {
+      id: id1,
+      error: undefined,
+      result: 'UNPUBLISHED',
+    },
+  });
+  t.is(await flushInboundQueue(1), 1);
+  t.like(wallet.getLatestUpdateRecord(), {
+    status: {
+      id: id2,
+      error: undefined,
+      result: 'UNPUBLISHED',
+    },
+  });
+});

Diff for: packages/boot/tools/supports.ts

@@ -334,11 +334,6 @@ export const makeSwingsetTestKit = async (
   };
   let ibcSequenceNonce = 0;
 
-  const addSequenceNonce = ({ packet }: IBCMethod<'sendPacket'>): IBCPacket => {
-    ibcSequenceNonce += 1;
-    return { ...packet, sequence: ibcSequenceNonce };
-  };
-
   /**
    * Adds the sequence so the bridge knows what response to connect it to.
    * Then queue it send it over the bridge over this returns.
@@ -358,14 +353,18 @@ export const makeSwingsetTestKit = async (
   };
 
   const inboundQueue: [bridgeId: BridgeIdValue, arg1: unknown][] = [];
+  /** Add a message that will be sent to the bridge by flushInboundQueue. */
+  const pushInbound = (bridgeId: BridgeIdValue, arg1: unknown) => {
+    inboundQueue.push([bridgeId, arg1]);
+  };
   /**
    * Like ackImmediately but defers in the inbound receiverAck
    * until `bridgeQueue()` is awaited.
    */
   const ackLater = (obj: IBCMethod<'sendPacket'>, ack: string) => {
     ibcSequenceNonce += 1;
     const msg = icaMocks.ackPacketEvent(obj, ibcSequenceNonce, ack);
-    inboundQueue.push([BridgeId.DIBC, msg]);
+    pushInbound(BridgeId.DIBC, msg);
     return msg.packet;
   };
 
@@ -437,10 +436,7 @@ export const makeSwingsetTestKit = async (
           case 'IBC_METHOD':
             switch (obj.method) {
               case 'startChannelOpenInit':
-                inboundQueue.push([
-                  BridgeId.DIBC,
-                  icaMocks.channelOpenAck(obj),
-                ]);
+                pushInbound(BridgeId.DIBC, icaMocks.channelOpenAck(obj));
                 return undefined;
               case 'sendPacket':
                 switch (obj.packet.data) {
@@ -619,36 +615,38 @@ export const makeSwingsetTestKit = async (
 
   const getCrankNumber = () => Number(kernelStorage.kvStore.get('crankNumber'));
 
-  const getOutboundMessages = (bridgeId: string) =>
-    harden([...outboundMessages.get(bridgeId)]);
-
-  /**
-   * @param {number} max the max number of messages to flush
-   * @returns {Promise<number>} the number of messages flushed
-   */
-  const flushInboundQueue = async (max: number = Number.POSITIVE_INFINITY) => {
-    console.log('🚽');
-    let i = 0;
-    for (i = 0; i < max; i += 1) {
-      const args = inboundQueue.shift();
-      if (!args) break;
-
-      await runUtils.queueAndRun(() => inbound(...args), true);
-    }
-    console.log('🧻');
-    return i;
+  const bridgeUtils = {
+    /** Immediately handle the inbound message */
+    inbound: bridgeInbound,
+    getOutboundMessages: (bridgeId: string) =>
+      harden([...outboundMessages.get(bridgeId)]),
+    getInboundQueueLength: () => inboundQueue.length,
+    /**
+     * @param {number} max the max number of messages to flush
+     * @returns {Promise<number>} the number of messages flushed
+     */
+    async flushInboundQueue(max: number = Number.POSITIVE_INFINITY) {
+      console.log('🚽');
+      let i = 0;
+      for (i = 0; i < max; i += 1) {
+        const args = inboundQueue.shift();
+        if (!args) break;
+
+        await runUtils.queueAndRun(() => inbound(...args), true);
+      }
+      console.log('🧻');
+      return i;
+    },
   };
 
   return {
     advanceTimeBy,
     advanceTimeTo,
+    bridgeUtils,
     buildProposal,
-    bridgeInbound,
     controller,
-    flushInboundQueue,
     evalProposal,
     getCrankNumber,
-    getOutboundMessages,
     jumpTimeTo,
     readLatest,
     runUtils,

Diff for: packages/builders/scripts/testing/restart-basic-flows.js

@@ -0,0 +1,100 @@
+/**
+ * @file This is for use in tests.
+ * Unlike most builder scripts, this one includes the proposal exports as well.
+ */
+import {
+  deeplyFulfilledObject,
+  makeTracer,
+  NonNullish,
+} from '@agoric/internal';
+import { E } from '@endo/far';
+
+/// <reference types="@agoric/vats/src/core/types-ambient"/>
+
+const trace = makeTracer('RestartBasicFlows', true);
+
+/**
+ * @import {start as StartFn} from '@agoric/orchestration/src/examples/basic-flows.contract.js';
+ */
+
+/**
+ * @param {BootstrapPowers} powers
+ */
+export const restartBasicFlows = async ({
+  consume: {
+    agoricNames,
+    board,
+    chainStorage,
+    chainTimerService,
+    cosmosInterchainService,
+    localchain,
+
+    contractKits,
+  },
+  instance: instances,
+}) => {
+  trace(restartBasicFlows.name);
+
+  // @ts-expect-error unknown instance
+  const instance = await instances.consume.basicFlows;
+  trace('instance', instance);
+  /** @type {StartedInstanceKit<StartFn>} */
+  const kit = /** @type {any} */ (await E(contractKits).get(instance));
+
+  const marshaller = await E(board).getReadonlyMarshaller();
+
+  const privateArgs = await deeplyFulfilledObject(
+    harden({
+      agoricNames,
+      localchain,
+      marshaller,
+      orchestrationService: cosmosInterchainService,
+      storageNode: E(NonNullish(await chainStorage)).makeChildNode(
+        'basicFlows',
+      ),
+      timerService: chainTimerService,
+    }),
+  );
+
+  await E(kit.adminFacet).restartContract(privateArgs);
+  trace('done');
+};
+harden(restartBasicFlows);
+
+export const getManifest = () => {
+  return {
+    manifest: {
+      [restartBasicFlows.name]: {
+        consume: {
+          agoricNames: true,
+          board: true,
+          chainStorage: true,
+          chainTimerService: true,
+          cosmosInterchainService: true,
+          localchain: true,
+
+          contractKits: true,
+        },
+        instance: {
+          consume: { basicFlows: true },
+        },
+      },
+    },
+  };
+};
+
+/** @type {import('@agoric/deploy-script-support/src/externalTypes.js').CoreEvalBuilder} */
+export const defaultProposalBuilder = async () =>
+  harden({
+    // Somewhat unorthodox, source the exports from this builder module
+    sourceSpec: '@agoric/builders/scripts/testing/restart-basic-flows.js',
+    getManifestCall: [getManifest.name],
+  });
+
+export default async (homeP, endowments) => {
+  // import dynamically so the module can work in CoreEval environment
+  const dspModule = await import('@agoric/deploy-script-support');
+  const { makeHelpers } = dspModule;
+  const { writeCoreEval } = await makeHelpers(homeP, endowments);
+  await writeCoreEval(restartBasicFlows.name, defaultProposalBuilder);
+};

Diff for: packages/orchestration/src/exos/local-orchestration-account.js

@@ -468,6 +468,9 @@ export const prepareLocalOrchestrationAccountKit = (
               this.facets.transferWatcher,
               { opts, amount, destination },
             );
+            // FIXME https://github.com/Agoric/agoric-sdk/issues/9783
+            // don't resolve the vow until the transfer is confirmed on remote
+            // and reject vow if the transfer fails
             return watch(transferV, this.facets.returnVoidWatcher);
           });
         },