retry install-bundle in multichain test (#11155)

closes: #9934

## Description

The **Fund Provision Pool** job is stable now, so the next one is flaking, **Override Chain Registry**.

```
----- Depl.5  5 installing bundles
----- Agd.3  2 $ agd tx swingset install-bundle @/tmp/contracts/b1-1fc262aefef4f13a10b2b116e07e4c8539619479f01fe6bd03fae243d2765c88b86054f15331c6d3c94c424d65a9fb0a3e8684d20ee4d31c6c9b705d6053e3c8.json --keyring-backend test --chain-id agoriclocal --from agoric1hm54wrxsv8e3pnw6lxj5lssfpexn48xtj6fhxw --broadcast-mode block --gas auto --gas-adjustment 1.4 --yes --output json
deploy failed, trying again (Error#1)
Error#1: Command failed: kubectl exec -i agoriclocal-genesis-0 -c validator --tty=false -- agd tx swingset install-bundle @/tmp/contracts/b1-1fc262aefef4f13a10b2b116e07e4c8539619479f01fe6bd03fae243d2765c88b86054f15331c6d3c94c[42](https://github.com/Agoric/agoric-sdk/actions/runs/13998860883/job/39200487363#step:12:43)4d65a9fb0a3e8684d20ee4d31c6c9b705d6053e3c8.json --keyring-backend test --chain-id agoriclocal --from agoric1hm54wrxsv8e3pnw6lxj5lssfpexn48xtj6fhxw --broadcast-mode block --gas auto --gas-adjustment 1.4 --yes --output json
Error: rpc error: code = Unknown desc = rpc error: code = Unknown desc = account sequence mismatch, expected 24, got 23: incorrect account sequence [agoric-labs/[email protected]/x/auth/ante/sigverify.go:269] With gas wanted: '18446744073709551615' and gas used: '1840763' : unknown request
```
It had a retry but wasn't waiting a block. This makes it wait a block.

### Security Considerations
It uses a global `fetch`. This is running in a Node env and just test code so I think it's warranted.

### Scaling Considerations
none

### Documentation Considerations
none

### Testing Considerations
If it passes, land it and hope it fixes the flake.

### Upgrade Considerations
n/a

Author: mergify[bot]

multichain-testing/scripts/fund-provision-pool.ts

@@ -37,21 +37,10 @@ async function fundProvisionPool(args: {
   console.log(`Funding provision pool at ${address} with ${amount}...`);
 
   // TODO: `@agoric/client-utils` should publish `waitForBlock`
-  const { waitForBlock } = (() => {
-    const delay = (ms: number): Promise<void> =>
-      new Promise(resolve => setTimeout(resolve, ms));
-    const explainDelay = (ms, info) => {
-      if (typeof info === 'object' && Object.keys(info).length > 0) {
-        console.log('delay', { ...info, delay: ms / 1000 }, '...');
-      }
-      return delay(ms);
-    };
-    const rpc = makeHttpClient(rpcUrl, fetch);
-    return makeBlockTool({
-      rpc,
-      delay: explainDelay,
-    });
-  })();
+  const { waitForBlock } = makeBlockTool({
+    rpc: makeHttpClient(rpcUrl, fetch),
+    delay: ms => new Promise(resolve => setTimeout(resolve, ms)),
+  });
 
   try {
     await waitForBlock(1);

multichain-testing/tools/e2e-tools.js

@@ -22,13 +22,12 @@ import { makeTracer } from '@agoric/internal';
 
 const trace = makeTracer('E2ET');
 
-// The default of 6 retries was failing.
-// XXX also tried 15, 30. There's probably something deeper to fix.
-const SMART_WALLET_PROVISION_RETRIES = 100;
 const PROVISIONING_POOL_ADDR = 'agoric1megzytg65cyrgzs6fvzxgrcqvwwl7ugpt62346';
 
 const BLD = '000000ubld';
 
+const rpcUrl = 'http://localhost:26657';
+
 export const txAbbr = tx => {
   const { txhash, code, height, gas_used } = tx;
 
@@ -106,20 +105,34 @@ export const makeBlockTool = ({ rpc, delay }) => {
  * @param {string} [opts.bundleId]
  */
 const installBundle = async (fullPath, opts) => {
-  const { id, agd, progress = console.log } = opts;
+  const { id, agd, delay, progress = console.log } = opts;
   const { chainId = 'agoriclocal', installer = 'faucet' } = opts;
-  const from = await agd.lookup(installer);
+  const from = agd.lookup(installer);
   // const explainDelay = (ms, info) => {
   //   progress('follow', { ...info, delay: ms / 1000 }, '...');
   //   return delay(ms);
   // };
   // const updates = follow('bundles', { delay: explainDelay });
   // await updates.next();
-  const tx = await agd.tx(['swingset', 'install-bundle', `@${fullPath}`], {
-    from,
-    chainId,
-    yes: true,
-  });
+  const installBundle = () =>
+    agd.tx(['swingset', 'install-bundle', `@${fullPath}`], {
+      from,
+      chainId,
+      yes: true,
+    });
+  let tx;
+  try {
+    tx = await installBundle();
+  } catch (err) {
+    console.warn('WARN: retrying failed install-bundle', err);
+    // TODO make available from client-utils
+    const { waitForBlock } = makeBlockTool({
+      rpc: makeHttpClient(rpcUrl, global.fetch),
+      delay,
+    });
+    await waitForBlock(1);
+    tx = await installBundle();
+  }
   assert(tx);
 
   progress({ id, installTx: tx.txhash, height: tx.height });
@@ -131,7 +144,7 @@ const installBundle = async (fullPath, opts) => {
   //   assert.equal(`b1-${confirm.endoZipBase64Sha512}`, opts.bundleId);
   // }
   // TODO: return block height at which confirm went into vstorage
-  return { tx, confirm: true };
+  return { tx, confirm: { installed: true } };
 };
 
 /**
@@ -241,9 +254,6 @@ const provisionSmartWalletAndMakeDriver = async (
       () => q.queryData(`published.wallet.${address}.current`),
       result => !!result,
       `wallet in vstorage ${address}`,
-      {
-        maxRetries: SMART_WALLET_PROVISION_RETRIES,
-      },
     );
     progress({
       provisioned: address,
@@ -549,7 +559,6 @@ export const makeE2ETools = async (
         // name,
         id: fullPath,
         installHeight: tx.height,
-        // @ts-expect-error confirm is a boolean?
         installed: confirm.installed,
       });
     }