chore: Add warning on the monitor's failure to update the latest bloc…

…k data
catalystdao · Jun 25, 2024 · 20a8ac1 · 20a8ac1
1 parent 8dbd169
commit 20a8ac1
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 0 deletions.
diff --git a/src/config/config.schema.ts b/src/config/config.schema.ts
@@ -100,6 +100,7 @@ const MONITOR_SCHEMA = {
             maximum: 120_000,   // 2 minutes
         },
         blockDelay: { $ref: "positive-number-schema" },
+        noBlockUpdateWarningInterval: { $ref: "positive-number-schema"},
     },
     additionalProperties: false
 }

diff --git a/src/config/config.types.ts b/src/config/config.types.ts
@@ -13,6 +13,7 @@ export interface GlobalConfig {
 export interface MonitorGlobalConfig {
   interval?: number;
   blockDelay?: number;
+  noBlockUpdateWarningInterval?: number;
 }
 
 export interface MonitorConfig extends MonitorGlobalConfig {}

diff --git a/src/monitor/monitor.service.ts b/src/monitor/monitor.service.ts
@@ -10,18 +10,21 @@ import { ChainConfig } from 'src/config/config.types';
 
 export const DEFAULT_MONITOR_INTERVAL = 5000;
 export const DEFAULT_MONITOR_BLOCK_DELAY = 1;
+export const DEFAULT_MONITOR_NO_BLOCK_UPDATE_WARNING_INTERVAL = 5 * 60000;
 
 
 interface DefaultMonitorWorkerData {
     interval: number,
     blockDelay: number,
+    noBlockUpdateWarningInterval: number,
 }
 
 export interface MonitorWorkerData {
     chainId: string,
     chainName: string,
     rpc: string,
     blockDelay: number,
+    noBlockUpdateWarningInterval: number,
     interval: number,
     loggerOptions: LoggerOptions
 }
@@ -79,10 +82,12 @@ export class MonitorService implements OnModuleInit {
 
         const blockDelay = globalMonitorConfig.blockDelay ?? DEFAULT_MONITOR_BLOCK_DELAY;
         const interval = globalMonitorConfig.interval ?? DEFAULT_MONITOR_INTERVAL;
+        const noBlockUpdateWarningInterval = globalMonitorConfig.noBlockUpdateWarningInterval ?? DEFAULT_MONITOR_NO_BLOCK_UPDATE_WARNING_INTERVAL;
 
         return {
             interval,
             blockDelay,
+            noBlockUpdateWarningInterval,
         }
     }
 
@@ -98,6 +103,7 @@ export class MonitorService implements OnModuleInit {
             rpc: chainConfig.rpc,
             blockDelay: chainMonitorConfig.blockDelay ?? defaultConfig.blockDelay,
             interval: chainMonitorConfig.interval ?? defaultConfig.interval,
+            noBlockUpdateWarningInterval: chainMonitorConfig.noBlockUpdateWarningInterval ?? defaultConfig.noBlockUpdateWarningInterval,
             loggerOptions: this.loggerService.loggerOptions
         };
     }

diff --git a/src/monitor/monitor.worker.ts b/src/monitor/monitor.worker.ts
@@ -18,6 +18,8 @@ class MonitorWorker {
     private portsCount = 0;
     private readonly ports: Record<number, MessagePort> = {};
 
+    private noBlockUpdateReferenceTimestamp = Date.now();
+
     private lastBroadcastBlockNumber = -1;
     private latestBlock: Block | null = null;
 
@@ -95,6 +97,22 @@ class MonitorWorker {
             try {
                 const newBlock = await this.provider.getBlock(-this.config.blockDelay);
                 if (!newBlock || newBlock.number <= this.lastBroadcastBlockNumber) {
+
+                    const noBlockUpdateWarningElapsedTime = Date.now()
+                        - this.noBlockUpdateReferenceTimestamp;
+
+                    if (noBlockUpdateWarningElapsedTime > this.config.noBlockUpdateWarningInterval) {
+                        this.logger.warn(
+                            {
+                                warningInterval: this.config.noBlockUpdateWarningInterval,
+                                latestBlockQuery: newBlock
+                            },
+                            `Failed to update block.`
+                        );
+
+                        // Avoid the logs from getting filled with warnings once 'noBlockUpdateWarningInterval' has elapsed
+                        this.noBlockUpdateReferenceTimestamp = Date.now();
+                    }
                     await wait(this.config.interval);
                     continue;
                 }
@@ -104,6 +122,8 @@ class MonitorWorker {
                 );
 
                 this.latestBlock = newBlock;
+                this.noBlockUpdateReferenceTimestamp = Date.now();
+
                 this.broadcastStatus();
             }
             catch (error) {