[Fixes] [Native Yield Automation Service] Observability improvements for Hoodi testing

.github/workflows/native-yield-automation-service-build-and-publish.yml

@@ -115,7 +115,7 @@ jobs:
           path: linea-native-yield-automation-service-docker-image.tar.gz
       - name: Build and push native-yield-automation-service image
         uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 #v6.18.0
-        if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }}
+        # if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }}
         with:
           context: ./
           file: ./native-yield-operations/automation-service/Dockerfile

native-yield-operations/automation-service/src/application/metrics/NativeYieldAutomationMetricsUpdater.ts

@@ -1,7 +1,7 @@
 import { IMetricsService } from "@consensys/linea-shared-utils";
 import {
   LineaNativeYieldAutomationServiceMetrics,
-  OperationTrigger,
+  OperationModeExecutionStatus,
 } from "../../core/metrics/LineaNativeYieldAutomationServiceMetrics.js";
 import { RebalanceDirection } from "../../core/entities/RebalanceRequirement.js";
 import { Address, Hex } from "viem";
@@ -96,16 +96,10 @@ export class NativeYieldAutomationMetricsUpdater implements INativeYieldAutomati
       ["vault_address"],
     );
 
-    this.metricsService.createCounter(
-      LineaNativeYieldAutomationServiceMetrics.OperationModeTriggerTotal,
-      "Operation mode triggers grouped by mode and triggers",
-      ["mode", "trigger"],
-    );
-
     this.metricsService.createCounter(
       LineaNativeYieldAutomationServiceMetrics.OperationModeExecutionTotal,
-      "Operation mode executions grouped by mode",
-      ["mode"],
+      "Operation mode executions grouped by mode and status",
+      ["mode", "status"],
     );
 
     this.metricsService.createHistogram(
@@ -291,26 +285,18 @@ export class NativeYieldAutomationMetricsUpdater implements INativeYieldAutomati
   }
 
   /**
-   * Increments the counter for operation mode triggers, grouped by mode and trigger type.
-   *
-   * @param {OperationMode} mode - The operation mode that was triggered.
-   * @param {OperationTrigger} trigger - The trigger that caused the mode to be activated.
-   */
-  public incrementOperationModeTrigger(mode: OperationMode, trigger: OperationTrigger): void {
-    this.metricsService.incrementCounter(LineaNativeYieldAutomationServiceMetrics.OperationModeTriggerTotal, {
-      mode,
-      trigger,
-    });
-  }
-
-  /**
-   * Increments the counter for operation mode executions, grouped by mode.
+   * Increments the counter for operation mode executions, grouped by mode and status.
    *
    * @param {OperationMode} mode - The operation mode that was executed.
+   * @param {OperationModeExecutionStatus} [status=OperationModeExecutionStatus.Success] - The execution status. Defaults to Success.
    */
-  public incrementOperationModeExecution(mode: OperationMode): void {
+  public incrementOperationModeExecution(
+    mode: OperationMode,
+    status: OperationModeExecutionStatus = OperationModeExecutionStatus.Success,
+  ): void {
     this.metricsService.incrementCounter(LineaNativeYieldAutomationServiceMetrics.OperationModeExecutionTotal, {
       mode,
+      status,
     });
   }
 

native-yield-operations/automation-service/src/application/metrics/__tests__/NativeYieldAutomationMetricsUpdater.test.ts

@@ -3,7 +3,7 @@ import { IMetricsService } from "@consensys/linea-shared-utils";
 import { NativeYieldAutomationMetricsUpdater } from "../NativeYieldAutomationMetricsUpdater.js";
 import {
   LineaNativeYieldAutomationServiceMetrics,
-  OperationTrigger,
+  OperationModeExecutionStatus,
 } from "../../../core/metrics/LineaNativeYieldAutomationServiceMetrics.js";
 import { RebalanceDirection } from "../../../core/entities/RebalanceRequirement.js";
 import { OperationMode } from "../../../core/enums/OperationModeEnums.js";
@@ -95,15 +95,10 @@ describe("NativeYieldAutomationMetricsUpdater", () => {
       "Lido fees paid by automation per vault",
       ["vault_address"],
     );
-    expect(metricsService.createCounter).toHaveBeenCalledWith(
-      LineaNativeYieldAutomationServiceMetrics.OperationModeTriggerTotal,
-      "Operation mode triggers grouped by mode and triggers",
-      ["mode", "trigger"],
-    );
     expect(metricsService.createCounter).toHaveBeenCalledWith(
       LineaNativeYieldAutomationServiceMetrics.OperationModeExecutionTotal,
-      "Operation mode executions grouped by mode",
-      ["mode"],
+      "Operation mode executions grouped by mode and status",
+      ["mode", "status"],
     );
     expect(metricsService.createHistogram).toHaveBeenCalledWith(
       LineaNativeYieldAutomationServiceMetrics.OperationModeExecutionDurationSeconds,
@@ -358,29 +353,42 @@ describe("NativeYieldAutomationMetricsUpdater", () => {
     });
   });
 
-  it("increments operation mode trigger counter", () => {
+  it("increments operation mode execution counter with default success status", () => {
     const metricsService = createMetricsServiceMock();
     const updater = new NativeYieldAutomationMetricsUpdater(metricsService);
     jest.clearAllMocks();
 
-    updater.incrementOperationModeTrigger(OperationMode.YIELD_REPORTING_MODE, OperationTrigger.TIMEOUT);
+    updater.incrementOperationModeExecution(OperationMode.OSSIFICATION_PENDING_MODE);
 
     expect(metricsService.incrementCounter).toHaveBeenCalledWith(
-      LineaNativeYieldAutomationServiceMetrics.OperationModeTriggerTotal,
-      { mode: OperationMode.YIELD_REPORTING_MODE, trigger: OperationTrigger.TIMEOUT },
+      LineaNativeYieldAutomationServiceMetrics.OperationModeExecutionTotal,
+      { mode: OperationMode.OSSIFICATION_PENDING_MODE, status: OperationModeExecutionStatus.Success },
     );
   });
 
-  it("increments operation mode execution counter", () => {
+  it("increments operation mode execution counter with explicit success status", () => {
     const metricsService = createMetricsServiceMock();
     const updater = new NativeYieldAutomationMetricsUpdater(metricsService);
     jest.clearAllMocks();
 
-    updater.incrementOperationModeExecution(OperationMode.OSSIFICATION_PENDING_MODE);
+    updater.incrementOperationModeExecution(OperationMode.YIELD_REPORTING_MODE, OperationModeExecutionStatus.Success);
+
+    expect(metricsService.incrementCounter).toHaveBeenCalledWith(
+      LineaNativeYieldAutomationServiceMetrics.OperationModeExecutionTotal,
+      { mode: OperationMode.YIELD_REPORTING_MODE, status: OperationModeExecutionStatus.Success },
+    );
+  });
+
+  it("increments operation mode execution counter with failure status", () => {
+    const metricsService = createMetricsServiceMock();
+    const updater = new NativeYieldAutomationMetricsUpdater(metricsService);
+    jest.clearAllMocks();
+
+    updater.incrementOperationModeExecution(OperationMode.YIELD_REPORTING_MODE, OperationModeExecutionStatus.Failure);
 
     expect(metricsService.incrementCounter).toHaveBeenCalledWith(
       LineaNativeYieldAutomationServiceMetrics.OperationModeExecutionTotal,
-      { mode: OperationMode.OSSIFICATION_PENDING_MODE },
+      { mode: OperationMode.YIELD_REPORTING_MODE, status: OperationModeExecutionStatus.Failure },
     );
   });
 

native-yield-operations/automation-service/src/application/metrics/__tests__/OperationModeMetricsRecorder.test.ts

@@ -34,7 +34,6 @@ const createMetricsUpdaterMock = (): jest.Mocked<INativeYieldAutomationMetricsUp
     addLidoFeesPaid: jest.fn(),
     incrementLidoVaultAccountingReport: jest.fn(),
     incrementOperationModeExecution: jest.fn(),
-    incrementOperationModeTrigger: jest.fn(),
     addValidatorPartialUnstakeAmount: jest.fn(),
     incrementValidatorExit: jest.fn(),
   }) as unknown as jest.Mocked<INativeYieldAutomationMetricsUpdater>;

native-yield-operations/automation-service/src/clients/__tests__/BeaconChainStakingClient.test.ts

@@ -37,7 +37,6 @@ const createMetricsUpdaterMock = () => {
     addNodeOperatorFeesPaid: jest.fn(),
     addLiabilitiesPaid: jest.fn(),
     addLidoFeesPaid: jest.fn(),
-    incrementOperationModeTrigger: jest.fn(),
     incrementOperationModeExecution: jest.fn(),
     recordOperationModeDuration: jest.fn(),
   };

native-yield-operations/automation-service/src/core/enums/OperationModeEnums.ts

@@ -2,4 +2,5 @@ export enum OperationMode {
   YIELD_REPORTING_MODE = "YIELD_REPORTING_MODE",
   OSSIFICATION_PENDING_MODE = "OSSIFICATION_PENDING_MODE",
   OSSIFICATION_COMPLETE_MODE = "OSSIFICATION_COMPLETE_MODE",
+  UNKNOWN = "UNKNOWN",
 }

native-yield-operations/automation-service/src/core/metrics/INativeYieldAutomationMetricsUpdater.ts

@@ -1,7 +1,7 @@
 import { Address, Hex } from "viem";
 import { RebalanceDirection } from "../entities/RebalanceRequirement.js";
 import { OperationMode } from "../enums/OperationModeEnums.js";
-import { OperationTrigger } from "./LineaNativeYieldAutomationServiceMetrics.js";
+import { OperationModeExecutionStatus } from "./LineaNativeYieldAutomationServiceMetrics.js";
 
 export interface INativeYieldAutomationMetricsUpdater {
   recordRebalance(direction: RebalanceDirection.STAKE | RebalanceDirection.UNSTAKE, amountGwei: number): void;
@@ -28,9 +28,7 @@ export interface INativeYieldAutomationMetricsUpdater {
 
   addLidoFeesPaid(vaultAddress: Address, amountGwei: number): void;
 
-  incrementOperationModeTrigger(mode: OperationMode, trigger: OperationTrigger): void;
-
-  incrementOperationModeExecution(mode: OperationMode): void;
+  incrementOperationModeExecution(mode: OperationMode, status?: OperationModeExecutionStatus): void;
 
   recordOperationModeDuration(mode: OperationMode, durationSeconds: number): void;
 }

native-yield-operations/automation-service/src/core/metrics/LineaNativeYieldAutomationServiceMetrics.ts

@@ -54,14 +54,10 @@ export enum LineaNativeYieldAutomationServiceMetrics {
   // N.B. Only accounts for payments by the automation service, but external actors can also trigger payment
   LidoFeesPaidTotal = "linea_native_yield_automation_service_lido_fees_paid_total",
 
-  // Counter that increments each time an operation mode is triggered.
+  // Counter that increments each time an operation mode completes execution.
   // Labels:
   // i.) `mode`
-  // i.) `operation_trigger` - VaultsReportDataUpdated_event vs timeout
-  OperationModeTriggerTotal = "linea_native_yield_automation_service_operation_mode_trigger_total",
-
-  // Counter that increments each time an operation mode completes execution.
-  // Single label `mode`
+  // ii.) `status` - OperationModeExecutionStatus.Success | OperationModeExecutionStatus.Failure
   OperationModeExecutionTotal = "linea_native_yield_automation_service_operation_mode_execution_total",
 
   // Histogram that tracks time for each operation mode run.
@@ -73,3 +69,8 @@ export enum OperationTrigger {
   VAULTS_REPORT_DATA_UPDATED_EVENT = "VAULTS_REPORT_DATA_UPDATED_EVENT",
   TIMEOUT = "TIMEOUT",
 }
+
+export enum OperationModeExecutionStatus {
+  Success = "success",
+  Failure = "failure",
+}

native-yield-operations/automation-service/src/services/OperationModeSelector.ts

@@ -5,6 +5,7 @@ import { IOperationModeSelector } from "../core/services/operation-mode/IOperati
 import { IOperationModeProcessor } from "../core/services/operation-mode/IOperationModeProcessor.js";
 import { INativeYieldAutomationMetricsUpdater } from "../core/metrics/INativeYieldAutomationMetricsUpdater.js";
 import { OperationMode } from "../core/enums/OperationModeEnums.js";
+import { OperationModeExecutionStatus } from "../core/metrics/LineaNativeYieldAutomationServiceMetrics.js";
 
 /**
  * Selects and executes the appropriate operation mode based on the yield provider's ossification state.
@@ -35,8 +36,7 @@ export class OperationModeSelector implements IOperationModeSelector {
     private readonly ossificationCompleteOperationModeProcessor: IOperationModeProcessor,
     private readonly yieldProvider: Address,
     private readonly contractReadRetryTimeMs: number,
-  ) {
-  }
+  ) {}
 
   /**
    * Starts the operation mode selection loop.
@@ -81,30 +81,44 @@ export class OperationModeSelector implements IOperationModeSelector {
    */
   private async selectOperationModeLoop(): Promise<void> {
     while (this.isRunning) {
+      let currentMode: OperationMode = OperationMode.UNKNOWN;
       try {
         const [isOssificationInitiated, isOssified] = await Promise.all([
           this.yieldManagerContractClient.isOssificationInitiated(this.yieldProvider),
           this.yieldManagerContractClient.isOssified(this.yieldProvider),
         ]);
 
         if (isOssified) {
+          currentMode = OperationMode.OSSIFICATION_COMPLETE_MODE;
           this.logger.info("Selected OSSIFICATION_COMPLETE_MODE");
           await this.ossificationCompleteOperationModeProcessor.process();
           this.logger.info("Completed OSSIFICATION_COMPLETE_MODE");
-          this.metricsUpdater.incrementOperationModeExecution(OperationMode.OSSIFICATION_COMPLETE_MODE);
+          this.metricsUpdater.incrementOperationModeExecution(
+            OperationMode.OSSIFICATION_COMPLETE_MODE,
+            OperationModeExecutionStatus.Success,
+          );
         } else if (isOssificationInitiated) {
+          currentMode = OperationMode.OSSIFICATION_PENDING_MODE;
           this.logger.info("Selected OSSIFICATION_PENDING_MODE");
           await this.ossificationPendingOperationModeProcessor.process();
           this.logger.info("Completed OSSIFICATION_PENDING_MODE");
-          this.metricsUpdater.incrementOperationModeExecution(OperationMode.OSSIFICATION_PENDING_MODE);
+          this.metricsUpdater.incrementOperationModeExecution(
+            OperationMode.OSSIFICATION_PENDING_MODE,
+            OperationModeExecutionStatus.Success,
+          );
         } else {
+          currentMode = OperationMode.YIELD_REPORTING_MODE;
           this.logger.info("Selected YIELD_REPORTING_MODE");
           await this.yieldReportingOperationModeProcessor.process();
           this.logger.info("Completed YIELD_REPORTING_MODE");
-          this.metricsUpdater.incrementOperationModeExecution(OperationMode.YIELD_REPORTING_MODE);
+          this.metricsUpdater.incrementOperationModeExecution(
+            OperationMode.YIELD_REPORTING_MODE,
+            OperationModeExecutionStatus.Success,
+          );
         }
       } catch (error) {
         this.logger.error(`selectOperationModeLoop error, retrying in ${this.contractReadRetryTimeMs}ms`, { error });
+        this.metricsUpdater.incrementOperationModeExecution(currentMode, OperationModeExecutionStatus.Failure);
         await wait(this.contractReadRetryTimeMs);
       }
     }