Merge pull request #7 from ajna-finance/42-crash-handlers

42: Handle crashes/rejections; use synchronous logs to prevent loss on crash

Author: ethanbennett

README.md

@@ -100,8 +100,8 @@ Due to LUP and HTP shifting dynamically with pool activity, the in-range boundar
       * `Move(fromBucket, toBucket, amount)` - vault function and event that shifts liquidity directly between buckets. The keeper uses it when consolidating out-of-range buckets into the `optimalBucket` without touching the Buffer.
       * `MoveFromBuffer(toBucket, amount)` - vault function and event that moves liquidity out of the Buffer into a bucket. The keeper calls this to drain Buffer surplus or to place funds into the `optimalBucket`.
       * `MoveToBuffer(fromBucket, amount)` - vault function and event that withdraws liquidity from a bucket into the Buffer; the keeper uses it to top up the Buffer or cover a deficit.
-    * Off-chain logs (via keeper `log` and `handleTransaction`)
-      * Run-level:
+    * Off-chain logs (pino-formatted JSON; filterable by event field)
+      * Logger events:
         * `keeper_run_succeeded` - final state with buffer total, buffer target, current price, and optimal bucket.
         * `keeper_run_failed` - run aborted with error details.
         * `keeper_stopping` - process shutdown (SIGINT/SIGTERM).
@@ -110,6 +110,11 @@ Due to LUP and HTP shifting dynamically with pool activity, the in-range boundar
         * `tx_failed` - failed tx with phase (`send`, `fail`, `revert`), hash, receipt, and context.
       * Warnings:
         * `buffer_imbalance` - emitted when the Buffer total does not match the computed bufferTarget after rebalancing (indicating a residual surplus/deficit).
+        * `price_query_failed` - query failed for the first of the two configured price feeds.
+      * Other errors:
+        * `subgraph_query_failed` - query for open auctions via configured subgraph threw an error.
+        * `uncaught_exception` - an unhandled error crashed the keeper process.
+        * `unhandled_rejection` - an unhandled promise rejection crashed the keeper process.
 
 ## Local Set Up
 

src/index.ts

@@ -1,4 +1,6 @@
+import { setUpCrashHandlers } from './utils/logger';
 import { startScheduler } from './utils/scheduler';
 import { run } from './keeper';
 
+setUpCrashHandlers();
 startScheduler(run);

src/oracle/price.ts

@@ -25,7 +25,7 @@ export async function getPrice(): Promise<bigint> {
       }
     } catch (err) {
       log.warn(
-        { event: 'price_query', err, tag },
+        { event: 'price_query_failed', err, tag },
         'primary price source failed, trying secondary source',
       );
     }

src/utils/logger.ts

@@ -1,7 +1,28 @@
 import pino from 'pino';
 import { env } from './env.ts';
 
-export const log = pino({
-  level: env.LOG_LEVEL ?? 'info',
-  redact: ['env.PRIVATE_KEY', 'env.ORACLE_API_KEY', 'env.RPC_URL'],
-});
+export const log = pino(
+  {
+    level: env.LOG_LEVEL ?? 'info',
+    redact: ['env.PRIVATE_KEY', 'env.ORACLE_API_KEY', 'env.RPC_URL'],
+  },
+  pino.destination({ sync: true }),
+);
+
+export function setUpCrashHandlers() {
+  process.on('uncaughtException', (err) => {
+    log.fatal({ event: 'uncaught_exception', err }, 'uncaughtException, process exiting');
+    process.exit(1);
+  });
+
+  process.on('unhandledRejection', (reason) => {
+    log.fatal(
+      {
+        event: 'unhandled_rejection',
+        err: reason instanceof Error ? reason : new Error(String(reason)),
+      },
+      'unhandledRejection, process exiting',
+    );
+    process.exit(1);
+  });
+}