From 374f59d82152f282d48d484a289f819a8b63c875 Mon Sep 17 00:00:00 2001 From: Dennis Fang Date: Fri, 15 Nov 2024 17:08:06 -0800 Subject: [PATCH] add metric to track insufficient balance errors --- hyperlane/relayer.go | 5 +++ .../order_fulfillment_handler.go | 8 +++- ordersettler/ordersettler.go | 6 +-- shared/metrics/metrics.go | 42 +++++++++++++------ 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/hyperlane/relayer.go b/hyperlane/relayer.go index 3c02d24..2bad479 100644 --- a/hyperlane/relayer.go +++ b/hyperlane/relayer.go @@ -4,8 +4,12 @@ import ( "encoding/hex" "errors" "fmt" +<<<<<<< HEAD "github.com/skip-mev/go-fast-solver/shared/metrics" "math/big" +======= + dbtypes "github.com/skip-mev/go-fast-solver/db" +>>>>>>> 42dca9e (add metric to track insufficient balance errors) "strings" @@ -150,6 +154,7 @@ func (r *relayer) Relay(ctx context.Context, originChainID string, initiateTxHas if err != nil { return "", "", fmt.Errorf("processing message on domain %s: %w", dispatch.DestinationDomain, err) } + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, destinationChainID, dbtypes.TxTypeHyperlaneMessageDelivery) destinationChainID, err = config.GetConfigReader(ctx).GetChainIDByHyperlaneDomain(dispatch.DestinationDomain) if err != nil { diff --git a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go index 780c2a2..1d9bc4c 100644 --- a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go +++ b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go @@ -205,7 +205,7 @@ func (r *orderFulfillmentHandler) FillOrder( } txHash, rawTx, _, err := destinationChainBridgeClient.FillOrder(ctx, order, destinationChainGatewayContractAddress) - metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.DestinationChainID, dbtypes.TxTypeOrderFill) if err != nil { return "", fmt.Errorf("filling order on destination chain at address %s: %w", destinationChainGatewayContractAddress, err) } @@ -235,6 +235,10 @@ func (r *orderFulfillmentHandler) checkOrderAssetBalance(ctx context.Context, de } if balance.Cmp(new(big.Int).SetUint64(transferAmount)) < 0 { lmt.Logger(ctx).Warn("insufficient balance", zap.String("balance", balance.String()), zap.Uint64("transferAmount", transferAmount)) + metrics.FromContext(ctx).ObserveInsufficientBalanceError( + destinationChainConfig.ChainID, + new(big.Int).Sub(new(big.Int).SetUint64(transferAmount), balance).Uint64(), + ) return false, nil } return true, nil @@ -429,7 +433,7 @@ func (r *orderFulfillmentHandler) InitiateTimeout(ctx context.Context, order db. } txHash, rawTx, _, err := destinationChainBridgeClient.InitiateTimeout(ctx, order, destinationChainGatewayContractAddress) - metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.DestinationChainID, dbtypes.TxTypeInitiateTimeout) if err != nil { return "", fmt.Errorf("initiating timeout: %w", err) } diff --git a/ordersettler/ordersettler.go b/ordersettler/ordersettler.go index 04ad993..6b6d47f 100644 --- a/ordersettler/ordersettler.go +++ b/ordersettler/ordersettler.go @@ -500,11 +500,7 @@ func (r *OrderSettler) SettleBatch(ctx context.Context, batch types.SettlementBa return "", fmt.Errorf("getting destination bridge client: %w", err) } txHash, rawTx, err := destinationBridgeClient.InitiateBatchSettlement(ctx, batch) - metrics.FromContext(ctx).IncTransactionSubmitted( - err == nil, - batch.SourceChainID(), - batch.DestinationChainID(), - ) + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, batch.DestinationChainID(), dbtypes.TxTypeSettlement) if err != nil { return "", fmt.Errorf("initiating batch settlement on chain %s: %w", batch.DestinationChainID(), err) } diff --git a/shared/metrics/metrics.go b/shared/metrics/metrics.go index 6e9d28a..2b0136b 100644 --- a/shared/metrics/metrics.go +++ b/shared/metrics/metrics.go @@ -18,11 +18,11 @@ const ( orderStatusLabel = "order_status" transferStatusLabel = "transfer_status" settlementStatusLabel = "settlement_status" - operationLabel = "operation" + transactionTypeLabel = "transaction_type" ) type Metrics interface { - IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) + IncTransactionSubmitted(success bool, chainID, transactionType string) IncTransactionVerified(success bool, chainID string) IncFillOrders(sourceChainID, destinationChainID, orderStatus string) @@ -43,6 +43,7 @@ type Metrics interface { ObserveTransferSizeOutOfRange(sourceChainID, destinationChainID string, amountExceededBy int64) ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBpsExceededBy int64) + ObserveInsufficientBalanceError(chainID string, amountInsufficientBy uint64) } type metricsContextKey struct{} @@ -78,10 +79,9 @@ type PromMetrics struct { hplCheckpointingErrors metrics.Counter hplLatency metrics.Histogram - transferSizeOutOfRange metrics.Histogram - feeBpsRejections metrics.Histogram - - databaseErrors metrics.Counter + transferSizeOutOfRange metrics.Histogram + feeBpsRejections metrics.Histogram + insufficientBalanceErrors metrics.Histogram } func NewPromMetrics() Metrics { @@ -162,16 +162,23 @@ func NewPromMetrics() Metrics { Help: "histogram of fee bps that were rejected for being too low", Buckets: []float64{1, 5, 10, 25, 50, 100, 200, 500, 1000}, }, []string{sourceChainIDLabel, destinationChainIDLabel}), - databaseErrors: prom.NewCounterFrom(stdprom.CounterOpts{ + insufficientBalanceErrors: prom.NewHistogramFrom(stdprom.HistogramOpts{ Namespace: "solver", - Name: "database_errors_total", - Help: "number of errors encountered when making database calls", - }, []string{}), + Name: "insufficient_balance_errors", + Help: "histogram of fill orders that exceeded available balance", + Buckets: []float64{ + 100000000, // 100 USDC + 1000000000, // 1,000 USDC + 10000000000, // 10,000 USDC + 100000000000, // 100,000 USDC + 1000000000000, // 1,000,000 USDC + }, + }, []string{chainIDLabel}), } } -func (m *PromMetrics) IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) { - m.totalTransactionSubmitted.With(successLabel, fmt.Sprint(success), sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID).Add(1) +func (m *PromMetrics) IncTransactionSubmitted(success bool, chainID, transactionType string) { + m.totalTransactionSubmitted.With(successLabel, fmt.Sprint(success), chainIDLabel, chainID, transactionTypeLabel, transactionType).Add(1) } func (m *PromMetrics) IncTransactionVerified(success bool, chainID string) { @@ -238,9 +245,18 @@ func (m *PromMetrics) ObserveFeeBpsRejection(sourceChainID, destinationChainID s ).Observe(float64(feeBps)) } +func (m *PromMetrics) ObserveInsufficientBalanceError(chainID string, amountInsufficientBy uint64) { + m.amountTransferSizeExceeded.With( + chainIDLabel, chainID, + ).Observe(float64(amountInsufficientBy)) +} + type NoOpMetrics struct{} -func (n NoOpMetrics) IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) { +func (n NoOpMetrics) ObserveInsufficientBalanceError(chainID string, amountInsufficientBy uint64) { +} + +func (n NoOpMetrics) IncTransactionSubmitted(success bool, chainID, transactionType string) { } func (n NoOpMetrics) IncTransactionVerified(success bool, chainID string) { }