Merge pull request #6 from ajna-finance/40-subgraph-failure

40: Optionally abort run on subgraph failure

Author: ethanbennett

.env.example

@@ -35,6 +35,10 @@ MIN_TIME_SINCE_BANKRUPTCY=259200
 # Defaults to 72 hours (in seconds).
 MAX_AUCTION_AGE=259200
 
+# Optional: this value determines whether the entire run exits if the subgraph query fails during the check for bad debt in the pool.
+# Defaults to false.
+EXIT_ON_SUBGRAPH_FAILURE=false
+
 # A CoinGecko API key is not strictly required for the rate of queries made by this keeper.
 #
 # If a key is defined, the tier is also required, since the header is different for each request (demo or pro).

README.md

@@ -43,10 +43,11 @@ Due to LUP and HTP shifting dynamically with pool activity, the in-range boundar
 | `MIN_MOVE_AMOUNT`              | Skip moves if bucket's quote token balance is below this amount (dust limit) - enforced by vault.    | Integer (WAD units)      | Optional                                       | 1,000,000        |
 | `MIN_TIME_SINCE_BANKRUPTCY`    | Minimum time since bucket bankruptcy to be considered valid. Abort keeper run if timestamp is between this value and current time.                     | Integer (seconds)        | Optional                                       | 259,200  (72h)    |
 | `MAX_AUCTION_AGE`              | Only consider auctions with bad debt if they are older than this value.                                    | Integer (seconds)        | Optional                                       | 259,200  (72h)    |
+| `EXIT_ON_SUBGRAPH_FAILURE` | Abort run if the subgraph query fails during the check for bad debt in the pool. | String (`true`/`false`) | Optional | `false`
 | `ORACLE_API_KEY`               | CoinGecko API key.                                                               | String                   | Optional                                       | None             |
 | `ORACLE_KEY_TIER`              | CoinGecko tier (`demo`, `pro`).                                                  | String                   | Conditional (if `ORACLE_API_KEY` set)          | None             |
 | `ONCHAIN_ORACLE_ADDRESS`       | Address of Chronicle on-chain oracle.                                            | Ethereum address (`0x…`) | Conditional (if `ONCHAIN_ORACLE_PRIMARY=true`) | None             |
-| `ONCHAIN_ORACLE_PRIMARY`       | Use on-chain oracle as primary instead of CoinGecko.                             | Boolean (`true/false`)   | Optional                                       | false            |
+| `ONCHAIN_ORACLE_PRIMARY`       | Use on-chain oracle as primary instead of CoinGecko.                             | String (`true`/`false`)   | Optional                                       | false            |
 | `ONCHAIN_ORACLE_MAX_STALENESS` | Max allowed age of on-chain price data.                                          | Integer (seconds)        | Conditional (if `ONCHAIN_ORACLE_PRIMARY=true`) | 43,200 (12h)     |
 
 

src/subgraph/poolHealth.ts

@@ -16,7 +16,8 @@ type LiquidationAuction = {
 
 export async function poolHasBadDebt(): Promise<boolean> {
   const unfilteredAuctions = await _getUnsettledAuctions();
-  const auctionsBeforeCutoff = _filterAuctions(unfilteredAuctions);
+  if (unfilteredAuctions === 'error') return true;
+  const auctionsBeforeCutoff = _filterAuctions(unfilteredAuctions as GetUnsettledAuctionsResponse);
 
   for (let i = 0; i < auctionsBeforeCutoff.length; i++) {
     const [kickTime, collateralRemaining, debtRemaining] = await getAuctionStatus(
@@ -29,7 +30,7 @@ export async function poolHasBadDebt(): Promise<boolean> {
   return false;
 }
 
-export async function _getUnsettledAuctions(): Promise<GetUnsettledAuctionsResponse> {
+export async function _getUnsettledAuctions(): Promise<GetUnsettledAuctionsResponse | string> {
   try {
     const poolAddress = (await getPoolAddress()).toLowerCase();
     const subgraphUrl = env.SUBGRAPH_URL;
@@ -51,10 +52,10 @@ export async function _getUnsettledAuctions(): Promise<GetUnsettledAuctionsRespo
   } catch (err) {
     log.error(
       { event: 'subgraph_query_failed', url: env.SUBGRAPH_URL, err },
-      'subgraph query failed, continuing with keeper run optimistically',
+      'subgraph query failed',
     );
 
-    return { liquidationAuctions: [] };
+    return env.EXIT_ON_SUBGRAPH_FAILURE ? 'error' : { liquidationAuctions: [] };
   }
 }
 

src/utils/env.ts

@@ -36,6 +36,13 @@ const minTimeSinceBankruptcy = process.env.MIN_TIME_SINCE_BANKRUPTCY ?? 259200;
 // Defaults to 72 hours (seconds)
 const maxAuctionAge = process.env.MAX_AUCTION_AGE ?? 259200;
 
+let exitOnSubgraphFailure;
+if (process.env.EXIT_ON_SUBGRAPH_FAILURE) {
+  exitOnSubgraphFailure = process.env.EXIT_ON_SUBGRAPH_FAILURE === 'true' ? true : false;
+} else {
+  exitOnSubgraphFailure = false;
+}
+
 export const env = {
   KEEPER_INTERVAL_MS: Number(process.env.KEEPER_INTERVAL_MS),
   VAULT_ADDRESS: process.env.VAULT_ADDRESS,
@@ -58,4 +65,5 @@ export const env = {
   SUBGRAPH_URL: process.env.SUBGRAPH_URL,
   MIN_TIME_SINCE_BANKRUPTCY: BigInt(minTimeSinceBankruptcy),
   MAX_AUCTION_AGE: Number(maxAuctionAge),
+  EXIT_ON_SUBGRAPH_FAILURE: exitOnSubgraphFailure,
 };

test/subgraph/poolHealth.test.ts

@@ -1,12 +1,11 @@
-import { describe, it, expect, expectTypeOf } from 'vitest';
+import { describe, it, expect } from 'vitest';
 import { _getUnsettledAuctions, _filterAuctions } from '../../src/subgraph/poolHealth';
 
 describe('subgraph query', () => {
   it('can query subgraph for unsettled auctions', async () => {
     const result = await _getUnsettledAuctions();
 
     expect(result).toHaveProperty('liquidationAuctions');
-    expectTypeOf(result.liquidationAuctions).toBeArray();
   });
 
   it('can filter out auctions newer than the minAge', () => {