add metric to track insufficient balance errors

hyperlane/relayer.go

@@ -4,8 +4,12 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+<<<<<<< HEAD
 	"github.com/skip-mev/go-fast-solver/shared/metrics"
 	"math/big"
+=======
+	dbtypes "github.com/skip-mev/go-fast-solver/db"
+>>>>>>> 42dca9e (add metric to track insufficient balance errors)
 
 	"strings"
 
@@ -150,6 +154,7 @@ func (r *relayer) Relay(ctx context.Context, originChainID string, initiateTxHas
 	if err != nil {
 		return "", "", fmt.Errorf("processing message on domain %s: %w", dispatch.DestinationDomain, err)
 	}
+	metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, destinationChainID, dbtypes.TxTypeHyperlaneMessageDelivery)
 
 	destinationChainID, err = config.GetConfigReader(ctx).GetChainIDByHyperlaneDomain(dispatch.DestinationDomain)
 	if err != nil {

orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go

@@ -205,7 +205,7 @@ func (r *orderFulfillmentHandler) FillOrder(
 	}
 
 	txHash, rawTx, _, err := destinationChainBridgeClient.FillOrder(ctx, order, destinationChainGatewayContractAddress)
-	metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID)
+	metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.DestinationChainID, dbtypes.TxTypeOrderFill)
 	if err != nil {
 		return "", fmt.Errorf("filling order on destination chain at address %s: %w", destinationChainGatewayContractAddress, err)
 	}
@@ -235,6 +235,10 @@ func (r *orderFulfillmentHandler) checkOrderAssetBalance(ctx context.Context, de
 	}
 	if balance.Cmp(new(big.Int).SetUint64(transferAmount)) < 0 {
 		lmt.Logger(ctx).Warn("insufficient balance", zap.String("balance", balance.String()), zap.Uint64("transferAmount", transferAmount))
+		metrics.FromContext(ctx).ObserveInsufficientBalanceError(
+			destinationChainConfig.ChainID,
+			new(big.Int).Sub(new(big.Int).SetUint64(transferAmount), balance).Uint64(),
+		)
 		return false, nil
 	}
 	return true, nil
@@ -429,7 +433,7 @@ func (r *orderFulfillmentHandler) InitiateTimeout(ctx context.Context, order db.
 	}
 
 	txHash, rawTx, _, err := destinationChainBridgeClient.InitiateTimeout(ctx, order, destinationChainGatewayContractAddress)
-	metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID)
+	metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.DestinationChainID, dbtypes.TxTypeInitiateTimeout)
 	if err != nil {
 		return "", fmt.Errorf("initiating timeout: %w", err)
 	}

ordersettler/ordersettler.go

@@ -500,11 +500,7 @@ func (r *OrderSettler) SettleBatch(ctx context.Context, batch types.SettlementBa
 		return "", fmt.Errorf("getting destination bridge client: %w", err)
 	}
 	txHash, rawTx, err := destinationBridgeClient.InitiateBatchSettlement(ctx, batch)
-	metrics.FromContext(ctx).IncTransactionSubmitted(
-		err == nil,
-		batch.SourceChainID(),
-		batch.DestinationChainID(),
-	)
+	metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, batch.DestinationChainID(), dbtypes.TxTypeSettlement)
 	if err != nil {
 		return "", fmt.Errorf("initiating batch settlement on chain %s: %w", batch.DestinationChainID(), err)
 	}

shared/metrics/metrics.go

@@ -18,11 +18,11 @@ const (
 	orderStatusLabel        = "order_status"
 	transferStatusLabel     = "transfer_status"
 	settlementStatusLabel   = "settlement_status"
-	operationLabel          = "operation"
+	transactionTypeLabel    = "transaction_type"
 )
 
 type Metrics interface {
-	IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string)
+	IncTransactionSubmitted(success bool, chainID, transactionType string)
 	IncTransactionVerified(success bool, chainID string)
 
 	IncFillOrders(sourceChainID, destinationChainID, orderStatus string)
@@ -43,6 +43,7 @@ type Metrics interface {
 
 	ObserveTransferSizeOutOfRange(sourceChainID, destinationChainID string, amountExceededBy int64)
 	ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBpsExceededBy int64)
+	ObserveInsufficientBalanceError(chainID string, amountInsufficientBy uint64)
 }
 
 type metricsContextKey struct{}
@@ -78,10 +79,9 @@ type PromMetrics struct {
 	hplCheckpointingErrors metrics.Counter
 	hplLatency             metrics.Histogram
 
-	transferSizeOutOfRange metrics.Histogram
-	feeBpsRejections       metrics.Histogram
-
-	databaseErrors metrics.Counter
+	transferSizeOutOfRange    metrics.Histogram
+	feeBpsRejections          metrics.Histogram
+	insufficientBalanceErrors metrics.Histogram
 }
 
 func NewPromMetrics() Metrics {
@@ -162,16 +162,23 @@ func NewPromMetrics() Metrics {
 			Help:      "histogram of fee bps that were rejected for being too low",
 			Buckets:   []float64{1, 5, 10, 25, 50, 100, 200, 500, 1000},
 		}, []string{sourceChainIDLabel, destinationChainIDLabel}),
-		databaseErrors: prom.NewCounterFrom(stdprom.CounterOpts{
+		insufficientBalanceErrors: prom.NewHistogramFrom(stdprom.HistogramOpts{
 			Namespace: "solver",
-			Name:      "database_errors_total",
-			Help:      "number of errors encountered when making database calls",
-		}, []string{}),
+			Name:      "insufficient_balance_errors",
+			Help:      "histogram of fill orders that exceeded available balance",
+			Buckets: []float64{
+				100000000,     // 100 USDC
+				1000000000,    // 1,000 USDC
+				10000000000,   // 10,000 USDC
+				100000000000,  // 100,000 USDC
+				1000000000000, // 1,000,000 USDC
+			},
+		}, []string{chainIDLabel}),
 	}
 }
 
-func (m *PromMetrics) IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) {
-	m.totalTransactionSubmitted.With(successLabel, fmt.Sprint(success), sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID).Add(1)
+func (m *PromMetrics) IncTransactionSubmitted(success bool, chainID, transactionType string) {
+	m.totalTransactionSubmitted.With(successLabel, fmt.Sprint(success), chainIDLabel, chainID, transactionTypeLabel, transactionType).Add(1)
 }
 
 func (m *PromMetrics) IncTransactionVerified(success bool, chainID string) {
@@ -238,9 +245,18 @@ func (m *PromMetrics) ObserveFeeBpsRejection(sourceChainID, destinationChainID s
 	).Observe(float64(feeBps))
 }
 
+func (m *PromMetrics) ObserveInsufficientBalanceError(chainID string, amountInsufficientBy uint64) {
+	m.amountTransferSizeExceeded.With(
+		chainIDLabel, chainID,
+	).Observe(float64(amountInsufficientBy))
+}
+
 type NoOpMetrics struct{}
 
-func (n NoOpMetrics) IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) {
+func (n NoOpMetrics) ObserveInsufficientBalanceError(chainID string, amountInsufficientBy uint64) {
+}
+
+func (n NoOpMetrics) IncTransactionSubmitted(success bool, chainID, transactionType string) {
 }
 func (n NoOpMetrics) IncTransactionVerified(success bool, chainID string) {
 }