return funds to LP on AdvanceFailed (#10980)

closes: #10969

## Description

Adds tests to reproduce #10969 and a fix.

### Security Considerations
none


### Scaling Considerations
none


### Documentation Considerations
none

### Testing Considerations
per se

### Upgrade Considerations
not yet deployed

Author: mergify[bot]

packages/fast-usdc/src/exos/advancer.js

@@ -32,12 +32,12 @@ import { makeFeeTools } from '../utils/fees.js';
  * @typedef {{
  *  chainHub: ChainHub;
  *  feeConfig: FeeConfig;
- *  localTransfer: ZoeTools['localTransfer'];
  *  log?: LogFn;
  *  statusManager: StatusManager;
  *  usdc: { brand: Brand<'nat'>; denom: Denom; };
  *  vowTools: VowTools;
  *  zcf: ZCF;
+ *  zoeTools: ZoeTools;
  * }} AdvancerKitPowers
  */
 
@@ -69,6 +69,16 @@ const AdvancerKitI = harden({
     ),
     onRejected: M.call(M.error(), AdvancerVowCtxShape).returns(M.undefined()),
   }),
+  withdrawHandler: M.interface('WithdrawHandlerI', {
+    onFulfilled: M.call(M.undefined(), {
+      advanceAmount: AnyNatAmountShape,
+      tmpReturnSeat: M.remotable(),
+    }).returns(M.undefined()),
+    onRejected: M.call(M.error(), {
+      advanceAmount: AnyNatAmountShape,
+      tmpReturnSeat: M.remotable(),
+    }).returns(M.undefined()),
+  }),
 });
 
 /**
@@ -98,12 +108,12 @@ export const prepareAdvancerKit = (
   {
     chainHub,
     feeConfig,
-    localTransfer,
     log = makeTracer('Advancer', true),
     statusManager,
     usdc,
     vowTools: { watch, when },
     zcf,
+    zoeTools: { localTransfer, withdrawToSeat },
   },
 ) => {
   assertAllDefined({
@@ -289,10 +299,55 @@ export const prepareAdvancerKit = (
          * @param {AdvancerVowCtx} ctx
          */
         onRejected(error, ctx) {
-          const { notifier } = this.state;
+          const { notifier, poolAccount } = this.state;
           log('Advance failed', error);
-          const { advanceAmount: _, ...restCtx } = ctx;
+          const { advanceAmount, ...restCtx } = ctx;
           notifier.notifyAdvancingResult(restCtx, false);
+          const { zcfSeat: tmpReturnSeat } = zcf.makeEmptySeatKit();
+          const withdrawV = withdrawToSeat(
+            // @ts-expect-error LocalAccountMethods vs OrchestrationAccount
+            poolAccount,
+            tmpReturnSeat,
+            harden({ USDC: advanceAmount }),
+          );
+          void watch(withdrawV, this.facets.withdrawHandler, {
+            advanceAmount,
+            tmpReturnSeat,
+          });
+        },
+      },
+      withdrawHandler: {
+        /**
+         *
+         * @param {undefined} result
+         * @param {{ advanceAmount: Amount<'nat'>; tmpReturnSeat: ZCFSeat; }} ctx
+         */
+        onFulfilled(result, { advanceAmount, tmpReturnSeat }) {
+          const { borrower } = this.state;
+          try {
+            borrower.returnToPool(tmpReturnSeat, advanceAmount);
+          } catch (e) {
+            // If we reach here, the unused advance funds will remain in `tmpReturnSeat`
+            // and must be retrieved from recovery sets.
+            log(
+              `🚨 return ${q(advanceAmount)} to pool failed. funds remain on "tmpReturnSeat"`,
+              e,
+            );
+          }
+          tmpReturnSeat.exit();
+        },
+        /**
+         * @param {Error} error
+         * @param {{ advanceAmount: Amount<'nat'>; tmpReturnSeat: ZCFSeat; }} ctx
+         */
+        onRejected(error, { advanceAmount, tmpReturnSeat }) {
+          log(
+            `🚨 withdraw ${q(advanceAmount)} from "poolAccount" to return to pool failed`,
+            error,
+          );
+          // If we reach here, the unused advance funds will remain in the `poolAccount`.
+          // A contract update will be required to return them to the LiquidityPool.
+          tmpReturnSeat.exit();
         },
       },
     },

packages/fast-usdc/src/exos/settler.js

@@ -362,7 +362,9 @@ export const prepareSettler = (
          * @param {EvmHash} txHash
          */
         onRejected(reason, txHash) {
-          log('⚠️ forward transfer rejected!', reason, txHash);
+          // funds remain in `settlementAccount` and must be recovered via a
+          // contract upgrade
+          log('🚨 forward transfer rejected!', reason, txHash);
           // update status manager, flagging a terminal state that needs to be
           // manual intervention or a code update to remediate
           statusManager.forwarded(txHash, false);

packages/fast-usdc/src/fast-usdc.contract.js

@@ -125,18 +125,18 @@ export const contract = async (zcf, privateArgs, zone, tools) => {
     chainHub,
   });
 
-  const { localTransfer } = makeZoeTools(zcf, vowTools);
+  const zoeTools = makeZoeTools(zcf, vowTools);
   const makeAdvancer = prepareAdvancer(zone, {
     chainHub,
     feeConfig,
-    localTransfer,
     usdc: harden({
       brand: terms.brands.USDC,
       denom: terms.usdcDenom,
     }),
     statusManager,
     vowTools,
     zcf,
+    zoeTools,
   });
 
   const makeFeedKit = prepareTransactionFeedKit(zone, zcf);

packages/fast-usdc/test/exos/advancer.test.ts

@@ -85,27 +85,33 @@ const createTestExtensions = (t, common: CommonSetup) => {
   });
 
   const localTransferVK = vowTools.makeVowKit<void>();
-  const resolveLocalTransferV = () => {
-    // pretend funds move from tmpSeat to poolAccount
-    localTransferVK.resolver.resolve();
-  };
-  const rejectLocalTransfeferV = () => {
+  // pretend funds move from tmpSeat to poolAccount
+  const resolveLocalTransferV = () => localTransferVK.resolver.resolve();
+  const rejectLocalTransfeferV = () =>
     localTransferVK.resolver.reject(
       new Error('One or more deposits failed: simulated error'),
     );
-  };
+  const withdrawToSeatVK = vowTools.makeVowKit<void>();
+  const resolveWithdrawToSeatV = () => withdrawToSeatVK.resolver.resolve();
+  const rejectWithdrawToSeatV = () =>
+    withdrawToSeatVK.resolver.reject(
+      new Error('One or more deposits failed: simulated error'),
+    );
   const mockZoeTools = Far('MockZoeTools', {
     localTransfer(...args: Parameters<ZoeTools['localTransfer']>) {
       trace('ZoeTools.localTransfer called with', args);
       return localTransferVK.vow;
     },
+    withdrawToSeat(...args: Parameters<ZoeTools['withdrawToSeat']>) {
+      trace('ZoeTools.withdrawToSeat called with', args);
+      return withdrawToSeatVK.vow;
+    },
   });
 
   const feeConfig = makeTestFeeConfig(usdc);
   const makeAdvancer = prepareAdvancer(contractZone.subZone('advancer'), {
     chainHub,
     feeConfig,
-    localTransfer: mockZoeTools.localTransfer,
     log,
     statusManager,
     usdc: harden({
@@ -115,6 +121,7 @@ const createTestExtensions = (t, common: CommonSetup) => {
     vowTools,
     // @ts-expect-error mocked zcf
     zcf: mockZCF,
+    zoeTools: mockZoeTools,
   });
 
   type NotifyArgs = Parameters<SettlerKit['notifier']['notifyAdvancingResult']>;
@@ -173,6 +180,8 @@ const createTestExtensions = (t, common: CommonSetup) => {
       mockNotifyF,
       resolveLocalTransferV,
       rejectLocalTransfeferV,
+      resolveWithdrawToSeatV,
+      rejectWithdrawToSeatV,
     },
     services: {
       advancer,
@@ -344,13 +353,13 @@ test('updates status to OBSERVED if makeChainAddress fails', async t => {
   ]);
 });
 
-test('calls notifyAdvancingResult (AdvancedFailed) on failed transfer', async t => {
+test('recovery behavior if Advance Fails (ADVANCE_FAILED)', async t => {
   const {
     bootstrap: { storage },
     extensions: {
       services: { advancer, feeTools },
-      helpers: { inspectLogs, inspectNotifyCalls },
-      mocks: { mockPoolAccount, resolveLocalTransferV },
+      helpers: { inspectBorrowerFacetCalls, inspectLogs, inspectNotifyCalls },
+      mocks: { mockPoolAccount, resolveLocalTransferV, resolveWithdrawToSeatV },
     },
     brands: { usdc },
   } = t.context;
@@ -394,6 +403,132 @@ test('calls notifyAdvancingResult (AdvancedFailed) on failed transfer', async t
       false, // this indicates transfer failed
     ],
   ]);
+
+  // simulate withdrawing `advanceAmount` from PoolAccount to tmpReturnSeat
+  resolveWithdrawToSeatV();
+  await eventLoopIteration();
+  const { returnToPool } = inspectBorrowerFacetCalls();
+  t.is(
+    returnToPool.length,
+    1,
+    'returnToPool is called after ibc transfer fails',
+  );
+  t.deepEqual(
+    returnToPool[0],
+    [
+      Far('MockZCFSeat', { exit: theExit }),
+      usdc.make(293999999n), // 300000000n net of fees
+    ],
+    'same amount borrowed is returned to LP',
+  );
+});
+
+// unexpected, terminal state. test that log('🚨') is called
+test('logs error if withdrawToSeat fails during AdvanceFailed recovery', async t => {
+  const {
+    extensions: {
+      services: { advancer },
+      helpers: { inspectLogs, inspectNotifyCalls },
+      mocks: { mockPoolAccount, resolveLocalTransferV, rejectWithdrawToSeatV },
+    },
+    brands: { usdc },
+  } = t.context;
+
+  const evidence = MockCctpTxEvidences.AGORIC_PLUS_OSMO();
+  void advancer.handleTransactionEvent({ evidence, risk: {} });
+
+  // pretend borrow succeeded and funds were depositing to the LCA
+  resolveLocalTransferV();
+  // pretend the IBC Transfer failed
+  mockPoolAccount.transferVResolver.reject(new Error('transfer failed'));
+  // pretend withdrawToSeat failed
+  rejectWithdrawToSeatV();
+  await eventLoopIteration();
+
+  t.deepEqual(inspectLogs(), [
+    ['decoded EUD: osmo183dejcnmkka5dzcu9xw6mywq0p2m5peks28men'],
+    ['Advance failed', Error('transfer failed')],
+    [
+      '🚨 withdraw {"brand":"[Alleged: USDC brand]","value":"[146999999n]"} from "poolAccount" to return to pool failed',
+      Error('One or more deposits failed: simulated error'),
+    ],
+  ]);
+
+  // ensure Settler is notified of failed advance
+  t.like(inspectNotifyCalls(), [
+    [
+      {
+        txHash: evidence.txHash,
+        forwardingAddress: evidence.tx.forwardingAddress,
+      },
+      false, // indicates transfer failed
+    ],
+  ]);
+});
+
+test('logs error if returnToPool fails during AdvanceFailed recovery', async t => {
+  const {
+    brands: { usdc },
+    extensions: {
+      services: { makeAdvancer },
+      helpers: { inspectLogs, inspectNotifyCalls },
+      mocks: {
+        mockPoolAccount,
+        mockNotifyF,
+        resolveLocalTransferV,
+        resolveWithdrawToSeatV,
+      },
+    },
+  } = t.context;
+
+  const mockBorrowerFacet = Far('LiquidityPool Borrow Facet', {
+    borrow: (seat: ZCFSeat, amount: NatAmount) => {
+      // note: will not be tracked by `inspectBorrowerFacetCalls`
+    },
+    returnToPool: (seat: ZCFSeat, amount: NatAmount) => {
+      throw new Error('returnToPool failed');
+    },
+  });
+
+  // make a new advancer that intentionally throws during returnToPool
+  const advancer = makeAdvancer({
+    borrower: mockBorrowerFacet,
+    notifier: mockNotifyF,
+    poolAccount: mockPoolAccount.account,
+    intermediateRecipient,
+    settlementAddress,
+  });
+
+  const evidence = MockCctpTxEvidences.AGORIC_PLUS_OSMO();
+  void advancer.handleTransactionEvent({ evidence, risk: {} });
+
+  // pretend borrow succeeded and funds were depositing to the LCA
+  resolveLocalTransferV();
+  // pretend the IBC Transfer failed
+  mockPoolAccount.transferVResolver.reject(new Error('transfer failed'));
+  // pretend withdrawToSeat succeeded
+  resolveWithdrawToSeatV();
+  await eventLoopIteration();
+
+  t.deepEqual(inspectLogs(), [
+    ['decoded EUD: osmo183dejcnmkka5dzcu9xw6mywq0p2m5peks28men'],
+    ['Advance failed', Error('transfer failed')],
+    [
+      '🚨 return {"brand":"[Alleged: USDC brand]","value":"[146999999n]"} to pool failed. funds remain on "tmpReturnSeat"',
+      Error('returnToPool failed'),
+    ],
+  ]);
+
+  // ensure Settler is notified of failed advance
+  t.like(inspectNotifyCalls(), [
+    [
+      {
+        txHash: evidence.txHash,
+        forwardingAddress: evidence.tx.forwardingAddress,
+      },
+      false, // indicates transfer failed
+    ],
+  ]);
 });
 
 test('updates status to OBSERVED if pre-condition checks fail', async t => {
@@ -785,8 +920,8 @@ test('notifies of advance failure if bank send fails', async t => {
   const {
     extensions: {
       services: { advancer },
-      helpers: { inspectLogs, inspectNotifyCalls },
-      mocks: { mockPoolAccount, resolveLocalTransferV },
+      helpers: { inspectLogs, inspectBorrowerFacetCalls, inspectNotifyCalls },
+      mocks: { mockPoolAccount, resolveLocalTransferV, resolveWithdrawToSeatV },
     },
     brands: { usdc },
   } = t.context;
@@ -820,4 +955,18 @@ test('notifies of advance failure if bank send fails', async t => {
       false, // indicates send failed
     ],
   ]);
+
+  // verify funds are returned to pool
+  resolveWithdrawToSeatV();
+  await eventLoopIteration();
+  const { returnToPool } = inspectBorrowerFacetCalls();
+  t.is(returnToPool.length, 1, 'returnToPool is called after bank send fails');
+  t.deepEqual(
+    returnToPool[0],
+    [
+      Far('MockZCFSeat', { exit: theExit }),
+      usdc.make(244999999n), // 250000000n net of fees
+    ],
+    'same amount borrowed is returned to LP',
+  );
 });