From 094e33d612d8cda846f2a42242bfdb0f043a31cc Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Mon, 28 Oct 2024 11:52:02 -0400 Subject: [PATCH 01/11] feat: add metrics --- cmd/solver/main.go | 15 +- fundrebalancer/fundrebalancer.go | 3 + fundrebalancer/transfermonitor.go | 27 +- hyperlane/relayer.go | 2 + hyperlane/relayer_runner.go | 11 + .../order_fulfillment_handler.go | 60 ++-- ordersettler/ordersettler.go | 44 ++- shared/metrics/metrics.go | 259 ++++++++++++------ transfermonitor/transfermonitor.go | 2 + txverifier/txverifier.go | 3 + 10 files changed, 279 insertions(+), 147 deletions(-) diff --git a/cmd/solver/main.go b/cmd/solver/main.go index 9cf5c0d..d991a0b 100644 --- a/cmd/solver/main.go +++ b/cmd/solver/main.go @@ -78,14 +78,13 @@ func main() { eg, ctx := errgroup.WithContext(ctx) - // Uncomment this section to run a prometheus server for metrics collection - //eg.Go(func() error { - // lmt.Logger(ctx).Info("Starting Prometheus") - // if err := metrics.StartPrometheus(ctx, fmt.Sprintf(cfg.Metrics.PrometheusAddress)); err != nil { - // return err - // } - // return nil - //}) + eg.Go(func() error { + lmt.Logger(ctx).Info("Starting Prometheus") + if err := metrics.StartPrometheus(ctx, fmt.Sprintf(cfg.Metrics.PrometheusAddress)); err != nil { + return err + } + return nil + }) eg.Go(func() error { transferMonitor := transfermonitor.NewTransferMonitor(db.New(dbConn), *quickStart) diff --git a/fundrebalancer/fundrebalancer.go b/fundrebalancer/fundrebalancer.go index babacf3..cbf4f42 100644 --- a/fundrebalancer/fundrebalancer.go +++ b/fundrebalancer/fundrebalancer.go @@ -4,6 +4,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + dbtypes "github.com/skip-mev/go-fast-solver/db" + "github.com/skip-mev/go-fast-solver/shared/metrics" "math/big" "os" "time" @@ -262,6 +264,7 @@ func (r *FundRebalancer) MoveFundsToChain( if remainingUSDCNeeded.Cmp(big.NewInt(0)) <= 0 { return hashes, totalUSDCcMoved, nil } + metrics.FromContext(ctx).IncFundsRebalanceTransfers(rebalanceFromChainID, rebalanceToChain, dbtypes.RebalanceTransactionStatusPending) } // we have moved all available funds from all available chains diff --git a/fundrebalancer/transfermonitor.go b/fundrebalancer/transfermonitor.go index 9c6c444..f863060 100644 --- a/fundrebalancer/transfermonitor.go +++ b/fundrebalancer/transfermonitor.go @@ -3,6 +3,7 @@ package fundrebalancer import ( "context" "fmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "time" "github.com/skip-mev/go-fast-solver/db" @@ -57,7 +58,7 @@ func (t *TransferTracker) UpdateTransfers(ctx context.Context) error { } for _, pendingTransfer := range pendingTransfers { - err := t.updateTransferStatus(ctx, pendingTransfer.ID, pendingTransfer.TxHash, pendingTransfer.SourceChainID) + err := t.updateTransferStatus(ctx, pendingTransfer.ID, pendingTransfer.TxHash, pendingTransfer.SourceChainID, pendingTransfer.DestinationChainID) if err != nil { lmt.Logger(ctx).Error( "error tracking transfer", @@ -74,10 +75,10 @@ func (t *TransferTracker) UpdateTransfers(ctx context.Context) error { return nil } -func (t *TransferTracker) updateTransferStatus(ctx context.Context, transferID int64, hash string, chainID string) error { - currentStatus, err := t.skipgo.Status(ctx, skipgo.TxHash(hash), chainID) +func (t *TransferTracker) updateTransferStatus(ctx context.Context, transferID int64, hash, sourceChainID, destinationChainID string) error { + currentStatus, err := t.skipgo.Status(ctx, skipgo.TxHash(hash), sourceChainID) if err != nil { - return fmt.Errorf("getting status for transaction %s on chain %s: %w", hash, chainID, err) + return fmt.Errorf("getting status for transaction %s on chain %s: %w", hash, sourceChainID, err) } // check if all transfers in the status are done @@ -95,12 +96,14 @@ func (t *TransferTracker) updateTransferStatus(ctx context.Context, transferID i lmt.Logger(ctx).Info( "waiting for transaction to complete", zap.String("latestState", string(latestState)), - zap.String("txnHash", string(hash)), - zap.String("chainID", chainID), + zap.String("txnHash", hash), + zap.String("sourceChainID", sourceChainID), + zap.String("destinationChainID", destinationChainID), ) return nil } + defer metrics.FromContext(ctx).DecFundsRebalanceTransfers(sourceChainID, destinationChainID, db.RebalanceTransactionStatusPending) // all transfers have finished, grab the first error if any var transferError string for _, transfer := range currentStatus.Transfers { @@ -114,7 +117,8 @@ func (t *TransferTracker) updateTransferStatus(ctx context.Context, transferID i lmt.Logger(ctx).Info( "rebalance transaction completed wtih an error", zap.String("txnHash", hash), - zap.String("chainID", chainID), + zap.String("sourceChainID", sourceChainID), + zap.String("destinationChainID", destinationChainID), zap.String("error", transferError), ) @@ -123,7 +127,8 @@ func (t *TransferTracker) updateTransferStatus(ctx context.Context, transferID i ID: transferID, }) if err != nil { - return fmt.Errorf("updating transfer status to failed for hash %s on chain %s: %w", hash, chainID, err) + metrics.FromContext(ctx).IncFundsRebalanceTransfers(sourceChainID, destinationChainID, db.RebalanceTransactionStatusFailed) + return fmt.Errorf("updating transfer status to failed for hash %s on chain %s: %w", hash, sourceChainID, err) } return nil @@ -132,15 +137,17 @@ func (t *TransferTracker) updateTransferStatus(ctx context.Context, transferID i lmt.Logger(ctx).Info( "rebalance transaction completed successfully", zap.String("txnHash", hash), - zap.String("chainID", chainID), + zap.String("sourceChainID", sourceChainID), + zap.String("destinationChainID", destinationChainID), ) + metrics.FromContext(ctx).IncFundsRebalanceTransfers(sourceChainID, destinationChainID, db.RebalanceTransactionStatusSuccess) err = t.database.UpdateTransferStatus(ctx, genDB.UpdateTransferStatusParams{ Status: db.RebalanceTransactionStatusSuccess, ID: transferID, }) if err != nil { - return fmt.Errorf("updating transfer status to completed for hash %s on chain %s: %w", hash, chainID, err) + return fmt.Errorf("updating transfer status to completed for hash %s on chain %s: %w", hash, sourceChainID, err) } return nil diff --git a/hyperlane/relayer.go b/hyperlane/relayer.go index 1086d46..ae541f0 100644 --- a/hyperlane/relayer.go +++ b/hyperlane/relayer.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "strings" @@ -166,6 +167,7 @@ func (r *relayer) checkpointAtIndex( continue } if err != nil { + metrics.FromContext(ctx).IncHyperlaneCheckpointingErrors() return types.MultiSigSignedCheckpoint{}, fmt.Errorf("fetching checkpoint at index %d: %w", index, err) } diff --git a/hyperlane/relayer_runner.go b/hyperlane/relayer_runner.go index 3503577..383f523 100644 --- a/hyperlane/relayer_runner.go +++ b/hyperlane/relayer_runner.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "time" dbtypes "github.com/skip-mev/go-fast-solver/db" @@ -125,6 +126,10 @@ func (r *RelayerRunner) checkHyperlaneTransferStatus(ctx context.Context, transf return false, fmt.Errorf("checking if message with id %s has been delivered: %w", transfer.MessageID, err) } if delivered { + metrics.FromContext(ctx).IncHyperlaneMessages(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusSuccess) + metrics.FromContext(ctx).DecHyperlaneMessages(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusSuccess) + metrics.FromContext(ctx).ObserveHyperlaneLatency(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusSuccess, time.Since(transfer.CreatedAt)) + if _, err := r.db.SetMessageStatus(ctx, db.SetMessageStatusParams{ TransferStatus: dbtypes.TransferStatusSuccess, SourceChainID: transfer.SourceChainID, @@ -147,6 +152,10 @@ func (r *RelayerRunner) checkHyperlaneTransferStatus(ctx context.Context, transf return false, fmt.Errorf("getting submitted txs by hyperlane transfer id %d: %w", transfer.ID, err) } if len(txs) > 0 { + metrics.FromContext(ctx).IncHyperlaneMessages(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusAbandoned) + metrics.FromContext(ctx).DecHyperlaneMessages(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusPending) + metrics.FromContext(ctx).ObserveHyperlaneLatency(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusAbandoned, time.Since(transfer.CreatedAt)) + // for now we will not attempt to submit the hyperlane message more than once. // this is to avoid the gas cost of repeatedly landing a failed hyperlane delivery tx. // in the future we may add more sophistication around retries @@ -198,6 +207,7 @@ func (r *RelayerRunner) findSettlementsToRelay(ctx context.Context) error { return fmt.Errorf("getting destination chainID by hyperlane domain %s: %w", dispatch.DestinationDomain, err) } + metrics.FromContext(ctx).IncHyperlaneMessages(pending.SourceChainID, destinationChainID, dbtypes.TransferStatusPending) if _, err := r.db.InsertHyperlaneTransfer(ctx, db.InsertHyperlaneTransferParams{ SourceChainID: pending.DestinationChainID, DestinationChainID: destinationChainID, @@ -238,6 +248,7 @@ func (r *RelayerRunner) findTimeoutsToRelay(ctx context.Context) error { return fmt.Errorf("getting destination chainID by hyperlane domain %s: %w", dispatch.DestinationDomain, err) } + metrics.FromContext(ctx).IncHyperlaneMessages(timeoutTx.ChainID, destinationChainID, dbtypes.TransferStatusPending) if _, err := r.db.InsertHyperlaneTransfer(ctx, db.InsertHyperlaneTransferParams{ SourceChainID: timeoutTx.ChainID, DestinationChainID: destinationChainID, diff --git a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go index fb31c76..fa97c04 100644 --- a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go +++ b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go @@ -58,6 +58,10 @@ func (r *orderFulfillmentHandler) UpdateFulfillmentStatus(ctx context.Context, o if err != nil { return "", fmt.Errorf("querying for order fill event on chainID %s at contract %s for order %s: %w", order.DestinationChainID, destinationChainGatewayContractAddress, order.OrderID, err) } else if fillTx != nil && filler != nil { + metrics.FromContext(ctx).IncFillOrders(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusFilled) + metrics.FromContext(ctx).DecFillOrders(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusPending) + metrics.FromContext(ctx).ObserveFillLatency(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusFilled, time.Since(order.CreatedAt)) + if _, err := r.db.SetFillTx(ctx, db.SetFillTxParams{ FillTx: sql.NullString{String: *fillTx, Valid: true}, Filler: sql.NullString{String: *filler, Valid: true}, @@ -79,6 +83,10 @@ func (r *orderFulfillmentHandler) UpdateFulfillmentStatus(ctx context.Context, o return "", fmt.Errorf("querying orderID %s has been refunded on chainID %s: %w", order.OrderID, order.SourceChainID, err) } if isRefunded { + metrics.FromContext(ctx).IncFillOrders(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusRefunded) + metrics.FromContext(ctx).DecFillOrders(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusExpiredPendingRefund) + metrics.FromContext(ctx).ObserveFillLatency(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusRefunded, time.Since(order.CreatedAt)) + _, err = r.db.SetRefundTx(ctx, db.SetRefundTxParams{ // TODO: do we really need to store the refund tx? to do this // we would need to have some order of indexer running for @@ -100,6 +108,9 @@ func (r *orderFulfillmentHandler) UpdateFulfillmentStatus(ctx context.Context, o return dbtypes.OrderStatusRefunded, nil } + metrics.FromContext(ctx).IncFillOrders(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusExpiredPendingRefund) + metrics.FromContext(ctx).DecFillOrders(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusPending) + if _, err := r.db.SetOrderStatus(ctx, db.SetOrderStatusParams{ SourceChainID: order.SourceChainID, OrderID: order.OrderID, @@ -182,10 +193,10 @@ func (r *orderFulfillmentHandler) FillOrder( } txHash, rawTx, _, err := destinationChainBridgeClient.FillOrder(ctx, order, destinationChainGatewayContractAddress) - metrics.FromContext(ctx).AddTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID, sourceChainConfig.ChainName, destinationChainConfig.ChainName, string(sourceChainConfig.Environment)) if err != nil { return "", fmt.Errorf("filling order on destination chain at address %s: %w", destinationChainBridgeClient, err) } + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) if _, err := r.db.InsertSubmittedTx(ctx, db.InsertSubmittedTxParams{ OrderID: sql.NullInt64{Int64: order.ID, Valid: true}, @@ -223,6 +234,10 @@ func (r *orderFulfillmentHandler) checkTransferSize(ctx context.Context, destina } if destinationChainConfig.MaxFillSize != nil && transferAmount > *destinationChainConfig.MaxFillSize { + metrics.FromContext(ctx).IncFillOrders(orderFill.SourceChainID, destinationChainConfig.ChainID, dbtypes.OrderStatusAbandoned) + metrics.FromContext(ctx).DecFillOrders(orderFill.SourceChainID, destinationChainConfig.ChainID, dbtypes.OrderStatusPending) + metrics.FromContext(ctx).ObserveFillLatency(orderFill.SourceChainID, orderFill.DestinationChainID, dbtypes.OrderStatusAbandoned, time.Since(orderFill.CreatedAt)) + _, err = r.db.SetOrderStatus(ctx, db.SetOrderStatusParams{ SourceChainID: orderFill.SourceChainID, OrderID: orderFill.OrderID, @@ -241,6 +256,11 @@ func (r *orderFulfillmentHandler) checkTransferSize(ctx context.Context, destina zap.String("orderAmountOut", orderFill.AmountOut), zap.Uint64("maxAllowedFillSize", *destinationChainConfig.MaxFillSize), ) + metrics.FromContext(ctx).ObserveTransferSizeExceeded( + orderFill.SourceChainID, + destinationChainConfig.ChainID, + transferAmount-*destinationChainConfig.MaxFillSize, + ) return false, nil } return true, nil @@ -255,7 +275,7 @@ func (r *orderFulfillmentHandler) checkFeeAmount(ctx context.Context, orderFill return false, fmt.Errorf("getting config for chainID %s: %w", orderFill.SourceChainID, err) } - isWithinBpsRange, err := IsWithinBpsRange(ctx, int64(sourceChainID.MinFeeBps), orderFill.AmountIn, orderFill.AmountOut) + isWithinBpsRange, bpsDiff, err := IsWithinBpsRange(ctx, int64(sourceChainID.MinFeeBps), orderFill.AmountIn, orderFill.AmountOut) if err != nil { return false, fmt.Errorf("checking if order fee for orderID %s is within min bps range: %w", orderFill.OrderID, err) } @@ -263,6 +283,10 @@ func (r *orderFulfillmentHandler) checkFeeAmount(ctx context.Context, orderFill return true, nil } + metrics.FromContext(ctx).IncFillOrders(orderFill.SourceChainID, orderFill.DestinationChainID, dbtypes.OrderStatusAbandoned) + metrics.FromContext(ctx).DecFillOrders(orderFill.SourceChainID, orderFill.DestinationChainID, dbtypes.OrderStatusPending) + metrics.FromContext(ctx).ObserveFillLatency(orderFill.SourceChainID, orderFill.DestinationChainID, dbtypes.OrderStatusAbandoned, time.Since(orderFill.CreatedAt)) + _, err = r.db.SetOrderStatus(ctx, db.SetOrderStatusParams{ SourceChainID: orderFill.SourceChainID, OrderID: orderFill.OrderID, @@ -282,27 +306,35 @@ func (r *orderFulfillmentHandler) checkFeeAmount(ctx context.Context, orderFill zap.Int("minFeeBps", sourceChainID.MinFeeBps), ) + metrics.FromContext(ctx).ObserveFeeBpsRejection( + orderFill.SourceChainID, + orderFill.DestinationChainID, + bpsDiff, + ) return false, nil } // IsWithinBpsRange returns true if the % change between amount in and amount -// out is >= min fee bps. -func IsWithinBpsRange(ctx context.Context, minFeeBps int64, amountIn, amountOut string) (bool, error) { +// out is >= min fee bps. If false, also returns the difference in bps. +func IsWithinBpsRange(ctx context.Context, minFeeBps int64, amountIn, amountOut string) (bool, int64, error) { minFee := big.NewInt(int64(minFeeBps)) in, ok := new(big.Int).SetString(amountIn, 10) if !ok { - return false, fmt.Errorf("converting amount in %s to *big.Int", amountIn) + return false, 0, fmt.Errorf("converting amount in %s to *big.Int", amountIn) } out, ok := new(big.Int).SetString(amountOut, 10) if !ok { - return false, fmt.Errorf("converting amount out %s to *big.Int", amountOut) + return false, 0, fmt.Errorf("converting amount out %s to *big.Int", amountOut) } minAcceptableFeeScaled := new(big.Int).Mul(minFee, in) feeAmount := new(big.Int).Sub(in, out) feeAmountScaled := new(big.Int).Mul(feeAmount, big.NewInt(10000)) - return feeAmountScaled.Cmp(minAcceptableFeeScaled) >= 0, nil + actualBps := new(big.Int).Div(feeAmountScaled, in).Int64() + bpsDiff := minFeeBps - actualBps + + return feeAmountScaled.Cmp(minAcceptableFeeScaled) >= 0, bpsDiff, nil } func (r *orderFulfillmentHandler) checkBlockConfirmations(ctx context.Context, sourceChainConfig config.ChainConfig, sourceChainBridgeClient cctp.BridgeClient, order db.Order) (confirmed bool, err error) { @@ -317,6 +349,10 @@ func (r *orderFulfillmentHandler) checkBlockConfirmations(ctx context.Context, s return false, err } if !exists { + metrics.FromContext(ctx).IncFillOrders(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusAbandoned) + metrics.FromContext(ctx).DecFillOrders(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusPending) + metrics.FromContext(ctx).ObserveFillLatency(order.SourceChainID, order.DestinationChainID, dbtypes.OrderStatusAbandoned, time.Since(order.CreatedAt)) + if _, err := r.db.SetOrderStatus(ctx, db.SetOrderStatusParams{ SourceChainID: order.SourceChainID, OrderID: order.OrderID, @@ -338,14 +374,6 @@ func (r *orderFulfillmentHandler) InitiateTimeout(ctx context.Context, order db. return fmt.Errorf("failed to get client: %w", err) } - sourceChainConfig, err := config.GetConfigReader(ctx).GetChainConfig(order.SourceChainID) - if err != nil { - return err - } - destinationChainConfig, err := config.GetConfigReader(ctx).GetChainConfig(order.DestinationChainID) - if err != nil { - return err - } destinationChainGatewayContractAddress, err := config.GetConfigReader(ctx).GetGatewayContractAddress(order.DestinationChainID) if err != nil { return err @@ -360,7 +388,7 @@ func (r *orderFulfillmentHandler) InitiateTimeout(ctx context.Context, order db. } txHash, rawTx, _, err := destinationChainBridgeClient.InitiateTimeout(ctx, order, destinationChainGatewayContractAddress) - metrics.FromContext(ctx).AddTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID, sourceChainConfig.ChainName, destinationChainConfig.ChainName, string(sourceChainConfig.Environment)) + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) if err != nil { return fmt.Errorf("initiating timeout: %w", err) } diff --git a/ordersettler/ordersettler.go b/ordersettler/ordersettler.go index 6597baf..e5f4086 100644 --- a/ordersettler/ordersettler.go +++ b/ordersettler/ordersettler.go @@ -167,6 +167,8 @@ func (r *OrderSettler) findNewSettlements(ctx context.Context) error { OrderID: fill.OrderID, SettlementStatus: dbtypes.SettlementStatusPending, }) + metrics.FromContext(ctx).IncOrderSettlements(sourceChainID, chain.ChainID, dbtypes.SettlementStatusPending) + if err != nil && !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("failed to insert settlement: %w", err) } @@ -314,14 +316,15 @@ func (r *OrderSettler) SettleBatch(ctx context.Context, batch types.SettlementBa return fmt.Errorf("getting destination bridge client: %w", err) } txHash, rawTx, err := destinationBridgeClient.InitiateBatchSettlement(ctx, batch) + metrics.FromContext(ctx).IncTransactionSubmitted( + err == nil, + batch.SourceChainID(), + batch.DestinationChainID(), + ) if err != nil { return fmt.Errorf("initiating batch settlement on chain %s: %w", batch.DestinationChainID(), err) } - if err = recordBatchSettlementSubmittedMetric(ctx, batch); err != nil { - return fmt.Errorf("recording batch settlement submitted metrics: %w", err) - } - err = r.db.InTx(ctx, func(ctx context.Context, q db.Querier) error { for _, settlement := range batch { settlementTx := db.SetInitiateSettlementTxParams{ @@ -359,30 +362,6 @@ func (r *OrderSettler) SettleBatch(ctx context.Context, batch types.SettlementBa return nil } -// recordBatchSettlementSubmittedMetric records a transaction submitted metric for a -// batch settlement -func recordBatchSettlementSubmittedMetric(ctx context.Context, batch types.SettlementBatch) error { - sourceChainConfig, err := batch.SourceChainConfig(ctx) - if err != nil { - return fmt.Errorf("getting source chain config for batch: %w", err) - } - destinationChainConfig, err := batch.DestinationChainConfig(ctx) - if err != nil { - return fmt.Errorf("getting destination chain config for batch: %w", err) - } - - metrics.FromContext(ctx).AddTransactionSubmitted( - err == nil, - batch.SourceChainID(), - batch.DestinationChainID(), - sourceChainConfig.ChainName, - destinationChainConfig.ChainName, - string(sourceChainConfig.Environment), - ) - - return nil -} - // verifyOrderSettlement checks if an order settlement tx is complete on chain // and updates the order settlement status in the db accordingly. func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db.OrderSettlement) error { @@ -404,6 +383,10 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db. return fmt.Errorf("failed to fetch message received event: %w", err) } else if failure != nil { lmt.Logger(ctx).Error("tx failed", zap.String("failure", failure.String())) + metrics.FromContext(ctx).IncOrderSettlements(settlement.SourceChainID, settlement.DestinationChainID, dbtypes.SettlementStatusFailed) + metrics.FromContext(ctx).DecOrderSettlements(settlement.SourceChainID, settlement.DestinationChainID, dbtypes.SettlementStatusPending) + metrics.FromContext(ctx).ObserveSettlementLatency(settlement.SourceChainID, settlement.DestinationChainID, dbtypes.SettlementStatusFailed, time.Since(settlement.CreatedAt)) + if _, err := r.db.SetSettlementStatus(ctx, db.SetSettlementStatusParams{ SourceChainID: settlement.SourceChainID, OrderID: settlement.OrderID, @@ -432,6 +415,10 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db. if settlementIsComplete, err := sourceBridgeClient.IsSettlementComplete(ctx, settlement.SourceChainGatewayContractAddress, settlement.OrderID); err != nil { return fmt.Errorf("failed to check if settlement is complete: %w", err) } else if settlementIsComplete { + metrics.FromContext(ctx).ObserveSettlementLatency(settlement.SourceChainID, settlement.DestinationChainID, settlement.SettlementStatus, time.Since(settlement.CreatedAt)) + metrics.FromContext(ctx).IncOrderSettlements(settlement.SourceChainID, settlement.DestinationChainID, dbtypes.SettlementStatusComplete) + metrics.FromContext(ctx).DecOrderSettlements(settlement.SourceChainID, settlement.DestinationChainID, dbtypes.SettlementStatusPending) + if _, err := r.db.SetSettlementStatus(ctx, db.SetSettlementStatusParams{ SourceChainID: settlement.SourceChainID, OrderID: settlement.OrderID, @@ -440,6 +427,7 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db. }); err != nil { return fmt.Errorf("failed to set relay status to complete: %w", err) } + return nil } return fmt.Errorf("settlement is not complete") diff --git a/shared/metrics/metrics.go b/shared/metrics/metrics.go index aefe375..12ab807 100644 --- a/shared/metrics/metrics.go +++ b/shared/metrics/metrics.go @@ -3,38 +3,45 @@ package metrics import ( "context" "fmt" - "math/big" "time" "github.com/go-kit/kit/metrics" prom "github.com/go-kit/kit/metrics/prometheus" stdprom "github.com/prometheus/client_golang/prometheus" - math2 "math" ) const ( - chainIDLabel = "chain_id" - gasBalanceLevelLabel = "gas_balance_level" - sourceChainIDLabel = "source_chain_id" - destinationChainIDLabel = "destination_chain_id" - successLabel = "success" - chainNameLabel = "chain_name" - sourceChainNameLabel = "source_chain_name" - destinationChainNameLabel = "destination_chain_name" - chainEnvironmentLabel = "chain_environment" - gasTokenSymbolLabel = "gas_token_symbol" + chainIDLabel = "chain_id" + sourceChainIDLabel = "source_chain_id" + destinationChainIDLabel = "destination_chain_id" + successLabel = "success" + orderStatusLabel = "order_status" + transferStatusLabel = "transfer_status" + settlementStatusLabel = "settlement_status" ) type Metrics interface { - SetGasBalance(string, string, string, string, big.Int, big.Int, big.Int, uint8) + IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) + IncTransactionVerified(success bool, chainID string) - AddSolverLoop() - SolverLoopLatency(time.Duration) - AddTransactionSubmitted(success bool, sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) - AddTransactionRetryAttempt(sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) - AddTransactionConfirmed(success bool, sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) + IncFillOrders(sourceChainID, destinationChainID, orderStatus string) + DecFillOrders(sourceChainID, destinationChainID, orderStatus string) + ObserveFillLatency(sourceChainID, destinationChainID string, orderStatus string, latency time.Duration) - FillLatency(sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string, latency time.Duration) + IncOrderSettlements(sourceChainID, destinationChainID, settlementStatus string) + DecOrderSettlements(sourceChainID, destinationChainID, settlementStatus string) + ObserveSettlementLatency(sourceChainID, destinationChainID string, settlementStatus string, latency time.Duration) + + IncFundsRebalanceTransfers(sourceChainID, destinationChainID string, transferStatus string) + DecFundsRebalanceTransfers(sourceChainID, destinationChainID string, transferStatus string) + + IncHyperlaneCheckpointingErrors() + IncHyperlaneMessages(sourceChainID, destinationChainID string, messageStatus string) + DecHyperlaneMessages(sourceChainID, destinationChainID string, messageStatus string) + ObserveHyperlaneLatency(sourceChainID, destinationChainID, transferStatus string, latency time.Duration) + + ObserveTransferSizeExceeded(sourceChainID, destinationChainID string, amountExceededBy uint64) + ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBpsExceededBy int64) } type metricsContextKey struct{} @@ -55,117 +62,199 @@ func FromContext(ctx context.Context) Metrics { var _ Metrics = (*PromMetrics)(nil) type PromMetrics struct { - gasBalance metrics.Gauge - totalSolverLoops metrics.Counter - latencyPerSolverLoop metrics.Histogram - totalTransactionSubmitted metrics.Counter - totalTransactionRetryAttempts metrics.Counter - totalTransactionsConfirmed metrics.Counter - latencyPerFill metrics.Histogram + totalTransactionSubmitted metrics.Counter + totalTransactionsVerified metrics.Counter + + fillOrders metrics.Gauge + fillLatency metrics.Histogram + + settlements metrics.Gauge + settlementLatency metrics.Histogram + + fundsRebalanceTransfers metrics.Gauge + + hplMessages metrics.Gauge + hplCheckpointingErrors metrics.Counter + hplLatency metrics.Histogram + + transferSizeExceeded metrics.Histogram + feeBpsRejections metrics.Histogram } func NewPromMetrics() Metrics { return &PromMetrics{ - gasBalance: prom.NewGaugeFrom(stdprom.GaugeOpts{ + fillOrders: prom.NewGaugeFrom(stdprom.GaugeOpts{ Namespace: "solver", - Name: "gas_balance_gauge", - Help: "gas balances, paginated by chain id and gas balance level", - }, []string{chainIDLabel, chainNameLabel, gasTokenSymbolLabel, chainEnvironmentLabel, gasBalanceLevelLabel}), - totalSolverLoops: prom.NewCounterFrom(stdprom.CounterOpts{ + Name: "fill_orders", + Help: "numbers of fill orders, paginated by source and destination chain, and status", + }, []string{sourceChainIDLabel, destinationChainIDLabel, orderStatusLabel}), + settlements: prom.NewGaugeFrom(stdprom.GaugeOpts{ Namespace: "solver", - Name: "total_solver_loops_counter", - Help: "number of solver loops", - }, []string{}), - latencyPerSolverLoop: prom.NewHistogramFrom(stdprom.HistogramOpts{ + Name: "settlements", + Help: "numbers of settlements intitiated, paginated by source and destination chain, and status", + }, []string{sourceChainIDLabel, destinationChainIDLabel, settlementStatusLabel}), + fundsRebalanceTransfers: prom.NewGaugeFrom(stdprom.GaugeOpts{ Namespace: "solver", - Name: "latency_per_solver_loop", - Help: "latency per solver loop in milliseconds", - Buckets: []float64{5, 10, 25, 50, 75, 100, 150, 200, 300, 500, 750, 1000, 1500, 3000, 5000, 10000, 20000}, - }, []string{}), + Name: "funds_rebalance_transfers", + Help: "numbers of funds rebalance transfers, paginated by source and destination chain, and status", + }, []string{sourceChainIDLabel, destinationChainIDLabel, transferStatusLabel}), + totalTransactionSubmitted: prom.NewCounterFrom(stdprom.CounterOpts{ Namespace: "solver", Name: "total_transactions_submitted_counter", Help: "number of transactions submitted, paginated by success status and source and destination chain id", - }, []string{successLabel, sourceChainIDLabel, destinationChainIDLabel, sourceChainNameLabel, destinationChainNameLabel, chainEnvironmentLabel}), - totalTransactionRetryAttempts: prom.NewCounterFrom(stdprom.CounterOpts{ + }, []string{successLabel, sourceChainIDLabel, destinationChainIDLabel}), + totalTransactionsVerified: prom.NewCounterFrom(stdprom.CounterOpts{ Namespace: "solver", - Name: "total_transaction_retry_attempts_counter", - Help: "number of transactions retried, paginated by source and destination chain id", - }, []string{sourceChainIDLabel, destinationChainIDLabel, sourceChainNameLabel, destinationChainNameLabel, chainEnvironmentLabel}), - totalTransactionsConfirmed: prom.NewCounterFrom(stdprom.CounterOpts{ - Namespace: "solver", - Name: "total_transactions_confirmed_counter", - Help: "number of transactions confirmed, paginated by success status and source and destination chain id", - }, []string{successLabel, sourceChainIDLabel, destinationChainIDLabel, sourceChainNameLabel, destinationChainNameLabel, chainEnvironmentLabel}), - latencyPerFill: prom.NewHistogramFrom(stdprom.HistogramOpts{ + Name: "total_transactions_verified_counter", + Help: "number of transactions verified, paginated by success status and chain id", + }, []string{successLabel, chainIDLabel}), + fillLatency: prom.NewHistogramFrom(stdprom.HistogramOpts{ Namespace: "solver", Name: "latency_per_fill", Help: "latency from source transaction to fill completion, paginated by source and destination chain id", Buckets: []float64{30, 60, 300, 600, 900, 1200, 1500, 1800, 2400, 3000, 3600}, - }, []string{sourceChainIDLabel, destinationChainIDLabel, sourceChainNameLabel, destinationChainNameLabel, chainEnvironmentLabel}), - } -} + }, []string{sourceChainIDLabel, destinationChainIDLabel, orderStatusLabel}), + settlementLatency: prom.NewHistogramFrom(stdprom.HistogramOpts{ + Namespace: "solver", + Name: "latency_per_settlement", + Help: "latency from source transaction to fill completion, paginated by source and destination chain id", + Buckets: []float64{30, 60, 300, 600, 900, 1200, 1500, 1800, 2400, 3000, 3600}, + }, []string{sourceChainIDLabel, destinationChainIDLabel, settlementStatusLabel}), + hplMessages: prom.NewGaugeFrom(stdprom.GaugeOpts{ + Namespace: "solver", + Name: "hyperlane_messages", + Help: "number of hyperlane messages, paginated by source and destination chain, and message status", + }, []string{sourceChainIDLabel, destinationChainIDLabel, transferStatusLabel}), -func (m *PromMetrics) SetGasBalance(chainID, chainName, gasTokenSymbol, chainEnvironment string, gasBalance, warningThreshold, criticalThreshold big.Int, gasTokenDecimals uint8) { - gasBalanceLevel := "ok" - if gasBalance.Cmp(&warningThreshold) < 0 { - gasBalanceLevel = "warning" - } - if gasBalance.Cmp(&criticalThreshold) < 0 { - gasBalanceLevel = "critical" + hplCheckpointingErrors: prom.NewCounterFrom(stdprom.CounterOpts{ + Namespace: "solver", + Name: "hyperlane_checkpointing_errors", + Help: "number of hyperlane checkpointing errors", + }, []string{}), + hplLatency: prom.NewHistogramFrom(stdprom.HistogramOpts{ + Namespace: "solver", + Name: "latency_per_hpl_message", + Help: "latency for hyperlane message relaying, paginated by status, source and destination chain id", + Buckets: []float64{30, 60, 300, 600, 900, 1200, 1500, 1800, 2400, 3000, 3600}, + }, []string{sourceChainIDLabel, destinationChainIDLabel, transferStatusLabel}), + transferSizeExceeded: prom.NewHistogramFrom(stdprom.HistogramOpts{ + Namespace: "solver", + Name: "transfer_size_exceeded", + Help: "histogram of transfer sizes that exceeded max fill size", + Buckets: []float64{ + 100000000, // 100 USDC + 1000000000, // 1,000 USDC + 10000000000, // 10,000 USDC + 100000000000, // 100,000 USDC + 1000000000000, // 1,000,000 USDC + }, + }, []string{sourceChainIDLabel, destinationChainIDLabel}), + feeBpsRejections: prom.NewHistogramFrom(stdprom.HistogramOpts{ + Namespace: "solver", + Name: "fee_bps_rejections", + Help: "histogram of fee bps that were rejected for being too low", + Buckets: []float64{1, 5, 10, 25, 50, 100, 200, 500, 1000}, + }, []string{sourceChainIDLabel, destinationChainIDLabel}), } - // We compare the gas balance against thresholds locally rather than in the grafana alert definition since - // the prometheus metric is exported as a float64 and the thresholds reach Wei amounts where precision is lost. - gasBalanceFloat, _ := gasBalance.Float64() - gasTokenAmount := gasBalanceFloat / (math2.Pow10(int(gasTokenDecimals))) - m.gasBalance.With(chainIDLabel, chainID, chainNameLabel, chainName, gasTokenSymbolLabel, gasTokenSymbol, chainEnvironmentLabel, chainEnvironment, gasBalanceLevelLabel, gasBalanceLevel).Set(gasTokenAmount) } -func (m *PromMetrics) AddSolverLoop() { - m.totalSolverLoops.Add(1) +func (m *PromMetrics) IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) { + m.totalTransactionSubmitted.With(successLabel, fmt.Sprint(success), sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID).Add(1) } -func (m *PromMetrics) SolverLoopLatency(latency time.Duration) { - m.latencyPerSolverLoop.Observe(float64(latency.Milliseconds())) +func (m *PromMetrics) IncTransactionVerified(success bool, chainID string) { + m.totalTransactionsVerified.With(successLabel, fmt.Sprint(success), chainIDLabel, chainID).Add(1) } -func (m *PromMetrics) AddTransactionSubmitted(success bool, sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) { - m.totalTransactionSubmitted.With(successLabel, fmt.Sprint(success), sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, sourceChainNameLabel, sourceChainName, destinationChainNameLabel, destinationChainName, chainEnvironmentLabel, chainEnvironment).Add(1) +func (m *PromMetrics) ObserveFillLatency(sourceChainID, destinationChainID, orderStatus string, latency time.Duration) { + m.fillLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, orderStatusLabel, orderStatus).Observe(latency.Seconds()) } -func (m *PromMetrics) AddTransactionRetryAttempt(sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) { - m.totalTransactionRetryAttempts.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, sourceChainNameLabel, sourceChainName, destinationChainNameLabel, destinationChainName, chainEnvironmentLabel, chainEnvironment).Add(1) +func (m *PromMetrics) ObserveSettlementLatency(sourceChainID, destinationChainID, settlementStatus string, latency time.Duration) { + m.settlementLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, settlementStatusLabel, settlementStatus).Observe(latency.Seconds()) } -func (m *PromMetrics) AddTransactionConfirmed(success bool, sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) { - m.totalTransactionsConfirmed.With(successLabel, fmt.Sprint(success), sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, sourceChainNameLabel, sourceChainName, destinationChainNameLabel, destinationChainName, chainEnvironmentLabel, chainEnvironment).Add(1) +func (m *PromMetrics) ObserveHyperlaneLatency(sourceChainID, destinationChainID, transferStatus string, latency time.Duration) { + m.hplLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, transferStatusLabel, transferStatus).Observe(latency.Seconds()) } -func (m *PromMetrics) FillLatency(sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string, latency time.Duration) { - m.latencyPerFill.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, sourceChainNameLabel, sourceChainName, destinationChainNameLabel, destinationChainName, chainEnvironmentLabel, chainEnvironment).Observe(float64(latency.Seconds())) +func (m *PromMetrics) IncFillOrders(sourceChainID, destinationChainID, orderStatus string) { + m.fillOrders.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, orderStatusLabel, orderStatus).Add(1) } -type NoOpMetrics struct{} +func (m *PromMetrics) DecFillOrders(sourceChainID, destinationChainID, orderStatus string) { + m.fillOrders.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, orderStatusLabel, orderStatus).Add(-1) +} -func (n NoOpMetrics) SetGasBalance(s string, s2 string, s3 string, s4 string, b big.Int, b2 big.Int, b3 big.Int, u uint8) { +func (m *PromMetrics) IncOrderSettlements(sourceChainID, destinationChainID, settlementStatus string) { + m.settlements.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, settlementStatusLabel, settlementStatus).Add(1) } -func (n NoOpMetrics) AddSolverLoop() {} +func (m *PromMetrics) DecOrderSettlements(sourceChainID, destinationChainID, settlementStatus string) { + m.settlements.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, settlementStatusLabel, settlementStatus).Add(-1) +} -func (n NoOpMetrics) SolverLoopLatency(duration time.Duration) {} +func (m *PromMetrics) IncFundsRebalanceTransfers(sourceChainID, destinationChainID, transferStatus string) { + m.fundsRebalanceTransfers.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, transferStatusLabel, transferStatus).Add(1) +} -func (n NoOpMetrics) AddTransactionSubmitted(success bool, sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) { +func (m *PromMetrics) DecFundsRebalanceTransfers(sourceChainID, destinationChainID, transferStatus string) { + m.fundsRebalanceTransfers.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, transferStatusLabel, transferStatus).Add(-1) } -func (n NoOpMetrics) AddTransactionRetryAttempt(sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) { +func (m *PromMetrics) IncHyperlaneCheckpointingErrors() { + m.hplCheckpointingErrors.Add(1) +} +func (m *PromMetrics) IncHyperlaneMessages(sourceChainID, destinationChainID, messageStatus string) { + m.hplMessages.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, messageStatus).Add(1) +} +func (m *PromMetrics) DecHyperlaneMessages(sourceChainID, destinationChainID, messageStatus string) { + m.hplMessages.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, messageStatus).Add(-1) } -func (n NoOpMetrics) AddTransactionConfirmed(success bool, sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string) { +func (m *PromMetrics) ObserveTransferSizeExceeded(sourceChainID, destinationChainID string, transferSize uint64) { + m.transferSizeExceeded.With( + sourceChainIDLabel, sourceChainID, + destinationChainIDLabel, destinationChainID, + ).Observe(float64(transferSize)) } -func (n NoOpMetrics) FillLatency(sourceChainID, destinationChainID, sourceChainName, destinationChainName, chainEnvironment string, latency time.Duration) { +func (m *PromMetrics) ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBps int64) { + m.feeBpsRejections.With( + sourceChainIDLabel, sourceChainID, + destinationChainIDLabel, destinationChainID, + ).Observe(float64(feeBps)) } +type NoOpMetrics struct{} + +func (n NoOpMetrics) IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) { +} +func (n NoOpMetrics) IncTransactionVerified(success bool, chainID string) { +} +func (n NoOpMetrics) ObserveFillLatency(sourceChainID, destinationChainID, orderStatus string, latency time.Duration) { +} +func (n NoOpMetrics) ObserveSettlementLatency(sourceChainID, destinationChainID, settlementStatus string, latency time.Duration) { +} +func (n NoOpMetrics) ObserveHyperlaneLatency(sourceChainID, destinationChainID, orderstatus string, latency time.Duration) { +} +func (n NoOpMetrics) IncFillOrders(sourceChainID, destinationChainID, orderStatus string) {} +func (n NoOpMetrics) DecFillOrders(sourceChainID, destinationChainID, orderStatus string) {} +func (n NoOpMetrics) IncOrderSettlements(sourceChainID, destinationChainID, settlementStatus string) { +} +func (n NoOpMetrics) DecOrderSettlements(sourceChainID, destinationChainID, settlementStatus string) { +} +func (n NoOpMetrics) IncFundsRebalanceTransfers(sourceChainID, destinationChainID, transferStatus string) { +} +func (n NoOpMetrics) DecFundsRebalanceTransfers(sourceChainID, destinationChainID, transferStatus string) { +} +func (n NoOpMetrics) IncHyperlaneCheckpointingErrors() {} +func (n NoOpMetrics) IncHyperlaneMessages(sourceChainID, destinationChainID, messageStatus string) {} +func (n NoOpMetrics) DecHyperlaneMessages(sourceChainID, destinationChainID, messageStatus string) {} +func (n NoOpMetrics) ObserveTransferSizeExceeded(sourceChainID, destinationChainID string, transferSize uint64) { +} +func (n NoOpMetrics) ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBps int64) {} func NewNoOpMetrics() Metrics { return &NoOpMetrics{} } diff --git a/transfermonitor/transfermonitor.go b/transfermonitor/transfermonitor.go index 44629b5..1884f2b 100644 --- a/transfermonitor/transfermonitor.go +++ b/transfermonitor/transfermonitor.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "encoding/hex" "fmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "math/big" "strings" "sync" @@ -142,6 +143,7 @@ func (t *TransferMonitor) Start(ctx context.Context) error { errorInsertingOrder = true break } + metrics.FromContext(ctx).IncFillOrders(order.ChainID, order.DestinationChainID, dbtypes.OrderStatusPending) } } lmt.Logger(ctx).Debug("num orders found while processing blocks", zap.Int("numOrders", len(orders))) diff --git a/txverifier/txverifier.go b/txverifier/txverifier.go index b6d2d32..558a0f6 100644 --- a/txverifier/txverifier.go +++ b/txverifier/txverifier.go @@ -6,6 +6,7 @@ import ( "fmt" dbtypes "github.com/skip-mev/go-fast-solver/db" "github.com/skip-mev/go-fast-solver/shared/clientmanager" + "github.com/skip-mev/go-fast-solver/shared/metrics" "time" coingecko2 "github.com/skip-mev/go-fast-solver/shared/clients/coingecko" @@ -102,6 +103,7 @@ func (r *TxVerifier) VerifyTx(ctx context.Context, submittedTx db.SubmittedTx) e return fmt.Errorf("failed to get tx result: %w", err) } else if failure != nil { lmt.Logger(ctx).Error("tx failed", zap.String("failure", failure.String())) + metrics.FromContext(ctx).IncTransactionVerified(false, submittedTx.ChainID) if _, err := r.db.SetSubmittedTxStatus(ctx, db.SetSubmittedTxStatusParams{ TxStatus: dbtypes.TxStatusFailed, TxHash: submittedTx.TxHash, @@ -112,6 +114,7 @@ func (r *TxVerifier) VerifyTx(ctx context.Context, submittedTx db.SubmittedTx) e } return fmt.Errorf("tx failed: %s", failure.String()) } else { + metrics.FromContext(ctx).IncTransactionVerified(true, submittedTx.ChainID) if _, err := r.db.SetSubmittedTxStatus(ctx, db.SetSubmittedTxStatusParams{ TxStatus: dbtypes.TxStatusSuccess, TxHash: submittedTx.TxHash, From fcf5b5761ee44c13e08af890d4a048334d99f87c Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Mon, 28 Oct 2024 12:07:28 -0400 Subject: [PATCH 02/11] fix test --- .../order_fulfillment_handler_test.go | 104 +++++++++++++++--- 1 file changed, 88 insertions(+), 16 deletions(-) diff --git a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler_test.go b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler_test.go index ce41787..ed1ca38 100644 --- a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler_test.go +++ b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler_test.go @@ -10,28 +10,100 @@ import ( func Test_CheckFeeAmount(t *testing.T) { tests := []struct { - Name string - MinFeeBps int64 - AmountIn string - AmountOut string - ShouldFill bool + Name string + MinFeeBps int64 + AmountIn string + AmountOut string + ShouldFill bool + ExpectedDiff int64 }{ - {Name: "100 bps min, 100 in 99 out", MinFeeBps: 100, AmountIn: "100", AmountOut: "99", ShouldFill: true}, - {Name: "200 bps min, 100 in 99 out", MinFeeBps: 200, AmountIn: "100", AmountOut: "99", ShouldFill: false}, - {Name: "100 bps min, 100 in 50 out", MinFeeBps: 100, AmountIn: "100", AmountOut: "50", ShouldFill: true}, - {Name: "1 bps min, 2 in 1 out", MinFeeBps: 1, AmountIn: "2", AmountOut: "1", ShouldFill: true}, - {Name: "100 bps min, 5mil in 4.99mil out", MinFeeBps: 100, AmountIn: "5000000", AmountOut: "4990000", ShouldFill: false}, - {Name: "100 bps min, 5mil in 4.95mil out", MinFeeBps: 100, AmountIn: "5000000", AmountOut: "4950000", ShouldFill: true}, - {Name: "200 bps min, 5mil in 4.95mil out", MinFeeBps: 200, AmountIn: "5000000", AmountOut: "4950000", ShouldFill: false}, - {Name: "200 bps min, 5mil in 4.95mil out", MinFeeBps: 200, AmountIn: "5000000", AmountOut: "4900000", ShouldFill: true}, - {Name: "0 bps min, 100 in 100 out", MinFeeBps: 0, AmountIn: "100", AmountOut: "100", ShouldFill: true}, - {Name: "0 bps min, 100 in 99 out", MinFeeBps: 0, AmountIn: "100", AmountOut: "99", ShouldFill: true}, + { + Name: "100 bps min, 100 in 99 out", + MinFeeBps: 100, + AmountIn: "100", + AmountOut: "99", + ShouldFill: true, + ExpectedDiff: 0, + }, + { + Name: "200 bps min, 100 in 99 out", + MinFeeBps: 200, + AmountIn: "100", + AmountOut: "99", + ShouldFill: false, + ExpectedDiff: 100, + }, + { + Name: "100 bps min, 100 in 50 out", + MinFeeBps: 100, + AmountIn: "100", + AmountOut: "50", + ShouldFill: true, + ExpectedDiff: -4900, + }, + { + Name: "1 bps min, 2 in 1 out", + MinFeeBps: 1, + AmountIn: "2", + AmountOut: "1", + ShouldFill: true, + ExpectedDiff: -4999, + }, + { + Name: "100 bps min, 5mil in 4.99mil out", + MinFeeBps: 100, + AmountIn: "5000000", + AmountOut: "4990000", + ShouldFill: false, + ExpectedDiff: 80, + }, + { + Name: "100 bps min, 5mil in 4.95mil out", + MinFeeBps: 100, + AmountIn: "5000000", + AmountOut: "4950000", + ShouldFill: true, + ExpectedDiff: 0, + }, + { + Name: "200 bps min, 5mil in 4.95mil out", + MinFeeBps: 200, + AmountIn: "5000000", + AmountOut: "4950000", + ShouldFill: false, + ExpectedDiff: 100, + }, + { + Name: "200 bps min, 5mil in 4.9mil out", + MinFeeBps: 200, + AmountIn: "5000000", + AmountOut: "4900000", + ShouldFill: true, + ExpectedDiff: 0, + }, + { + Name: "0 bps min, 100 in 100 out", + MinFeeBps: 0, + AmountIn: "100", + AmountOut: "100", + ShouldFill: true, + ExpectedDiff: 0, + }, + { + Name: "0 bps min, 100 in 99 out", + MinFeeBps: 0, + AmountIn: "100", + AmountOut: "99", + ShouldFill: true, + ExpectedDiff: -100, + }, } for _, tt := range tests { t.Run(tt.Name, func(t *testing.T) { - shouldFill, err := handler.IsWithinBpsRange(context.Background(), tt.MinFeeBps, tt.AmountIn, tt.AmountOut) + shouldFill, bpsDiff, err := handler.IsWithinBpsRange(context.Background(), tt.MinFeeBps, tt.AmountIn, tt.AmountOut) assert.NoError(t, err) assert.Equal(t, tt.ShouldFill, shouldFill) + assert.Equal(t, tt.ExpectedDiff, bpsDiff, "BPS difference mismatch") }) } } From 8f5ee975e3340565cc07204b6323f37446328768 Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Mon, 28 Oct 2024 15:31:39 -0400 Subject: [PATCH 03/11] add db metrics --- db/types.go | 4 ++++ hyperlane/relayer_runner.go | 11 +++++++++- .../order_fulfillment_handler.go | 20 ++++++++++++++----- orderfulfiller/orderfulfiller.go | 3 +++ ordersettler/ordersettler.go | 8 ++++++++ shared/metrics/metrics.go | 15 ++++++++++++++ transfermonitor/transfermonitor.go | 6 +++++- txverifier/txverifier.go | 6 +++++- 8 files changed, 65 insertions(+), 8 deletions(-) diff --git a/db/types.go b/db/types.go index 948f5c3..07b99f3 100644 --- a/db/types.go +++ b/db/types.go @@ -29,4 +29,8 @@ const ( TransferStatusPending string = "PENDING" TransferStatusSuccess string = "SUCCESS" TransferStatusAbandoned string = "ABANDONED" + + GET string = "GET" + INSERT string = "INSERT" + UPDATE string = "UPDATE" ) diff --git a/hyperlane/relayer_runner.go b/hyperlane/relayer_runner.go index 383f523..65a731f 100644 --- a/hyperlane/relayer_runner.go +++ b/hyperlane/relayer_runner.go @@ -5,9 +5,10 @@ import ( "database/sql" "errors" "fmt" - "github.com/skip-mev/go-fast-solver/shared/metrics" "time" + "github.com/skip-mev/go-fast-solver/shared/metrics" + dbtypes "github.com/skip-mev/go-fast-solver/db" "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/shared/config" @@ -66,6 +67,7 @@ func (r *RelayerRunner) Run(ctx context.Context) error { // grab all pending hyperlane transfers from the db transfers, err := r.db.GetAllHyperlaneTransfersWithTransferStatus(ctx, dbtypes.TransferStatusPending) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return fmt.Errorf("getting pending hyperlane transfers: %w", err) } @@ -102,6 +104,7 @@ func (r *RelayerRunner) Run(ctx context.Context) error { TxType: dbtypes.TxTypeHyperlaneMessageDelivery, TxStatus: dbtypes.TxStatusPending, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) lmt.Logger(ctx).Error( "error inserting submitted tx for hyperlane transfer", zap.Error(err), @@ -136,6 +139,7 @@ func (r *RelayerRunner) checkHyperlaneTransferStatus(ctx context.Context, transf DestinationChainID: transfer.DestinationChainID, MessageID: transfer.MessageID, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return false, fmt.Errorf("setting message status to success: %w", err) } lmt.Logger(ctx).Info( @@ -149,6 +153,7 @@ func (r *RelayerRunner) checkHyperlaneTransferStatus(ctx context.Context, transf txs, err := r.db.GetSubmittedTxsByHyperlaneTransferId(ctx, sql.NullInt64{Int64: transfer.ID, Valid: true}) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return false, fmt.Errorf("getting submitted txs by hyperlane transfer id %d: %w", transfer.ID, err) } if len(txs) > 0 { @@ -180,6 +185,7 @@ func (r *RelayerRunner) findSettlementsToRelay(ctx context.Context) error { // settlements that should be relayed pendingSettlements, err := r.db.GetAllOrderSettlementsWithSettlementStatus(ctx, dbtypes.SettlementStatusSettlementInitiated) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return fmt.Errorf("getting pending order settlements: %w", err) } @@ -215,6 +221,7 @@ func (r *RelayerRunner) findSettlementsToRelay(ctx context.Context) error { MessageSentTx: pending.InitiateSettlementTx.String, TransferStatus: dbtypes.TransferStatusPending, }); err != nil && !errors.Is(err, sql.ErrNoRows) { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) return fmt.Errorf("inserting hyperlane transfer: %w", err) } } @@ -229,6 +236,7 @@ func (r *RelayerRunner) findTimeoutsToRelay(ctx context.Context) error { TxType: dbtypes.TxTypeInitiateTimeout, }) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return fmt.Errorf("getting submitted txs for expired orders pending refunds: %w", err) } @@ -256,6 +264,7 @@ func (r *RelayerRunner) findTimeoutsToRelay(ctx context.Context) error { MessageSentTx: timeoutTx.TxHash, TransferStatus: dbtypes.TransferStatusPending, }); err != nil && !errors.Is(err, sql.ErrNoRows) { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) return fmt.Errorf("inserting hyperlane transfer: %w", err) } } diff --git a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go index fa97c04..c3708cf 100644 --- a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go +++ b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go @@ -70,6 +70,7 @@ func (r *orderFulfillmentHandler) UpdateFulfillmentStatus(ctx context.Context, o SourceChainGatewayContractAddress: order.SourceChainGatewayContractAddress, OrderStatus: dbtypes.OrderStatusFilled, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return "", err } return dbtypes.OrderStatusFilled, nil @@ -102,6 +103,7 @@ func (r *orderFulfillmentHandler) UpdateFulfillmentStatus(ctx context.Context, o OrderStatus: dbtypes.OrderStatusRefunded, }) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return "", fmt.Errorf("setting refund tx for orderID %s: %w", order.OrderID, err) } @@ -117,6 +119,7 @@ func (r *orderFulfillmentHandler) UpdateFulfillmentStatus(ctx context.Context, o SourceChainGatewayContractAddress: order.SourceChainGatewayContractAddress, OrderStatus: dbtypes.OrderStatusExpiredPendingRefund, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return "", err } return dbtypes.OrderStatusExpiredPendingRefund, nil @@ -180,6 +183,7 @@ func (r *orderFulfillmentHandler) FillOrder( OrderID: sql.NullInt64{Int64: order.ID, Valid: true}, TxType: dbtypes.TxTypeOrderFill, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return "", fmt.Errorf("failed to get submitted txs: %w", err) } else if len(submittedTxs) > 0 { // TODO will want to add some retry logic where even if this is > 0, we want to execute an order fill return "", nil @@ -206,6 +210,7 @@ func (r *orderFulfillmentHandler) FillOrder( TxType: dbtypes.TxTypeOrderFill, TxStatus: dbtypes.TxStatusPending, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) return "", fmt.Errorf("failed to insert raw tx %w", err) } @@ -237,6 +242,11 @@ func (r *orderFulfillmentHandler) checkTransferSize(ctx context.Context, destina metrics.FromContext(ctx).IncFillOrders(orderFill.SourceChainID, destinationChainConfig.ChainID, dbtypes.OrderStatusAbandoned) metrics.FromContext(ctx).DecFillOrders(orderFill.SourceChainID, destinationChainConfig.ChainID, dbtypes.OrderStatusPending) metrics.FromContext(ctx).ObserveFillLatency(orderFill.SourceChainID, orderFill.DestinationChainID, dbtypes.OrderStatusAbandoned, time.Since(orderFill.CreatedAt)) + metrics.FromContext(ctx).ObserveTransferSizeExceeded( + orderFill.SourceChainID, + destinationChainConfig.ChainID, + transferAmount-*destinationChainConfig.MaxFillSize, + ) _, err = r.db.SetOrderStatus(ctx, db.SetOrderStatusParams{ SourceChainID: orderFill.SourceChainID, @@ -246,6 +256,7 @@ func (r *orderFulfillmentHandler) checkTransferSize(ctx context.Context, destina OrderStatusMessage: sql.NullString{String: "amount exceeds max fill size", Valid: true}, }) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return false, fmt.Errorf("failed to set fill status to abandoned: %w", err) } @@ -256,11 +267,6 @@ func (r *orderFulfillmentHandler) checkTransferSize(ctx context.Context, destina zap.String("orderAmountOut", orderFill.AmountOut), zap.Uint64("maxAllowedFillSize", *destinationChainConfig.MaxFillSize), ) - metrics.FromContext(ctx).ObserveTransferSizeExceeded( - orderFill.SourceChainID, - destinationChainConfig.ChainID, - transferAmount-*destinationChainConfig.MaxFillSize, - ) return false, nil } return true, nil @@ -295,6 +301,7 @@ func (r *orderFulfillmentHandler) checkFeeAmount(ctx context.Context, orderFill OrderStatusMessage: sql.NullString{String: fmt.Sprintf("solver fee for order below configured min fee bps of %d", sourceChainID.MinFeeBps), Valid: true}, }) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return false, fmt.Errorf("failed to set fill status to abandoned: %w", err) } @@ -360,6 +367,7 @@ func (r *orderFulfillmentHandler) checkBlockConfirmations(ctx context.Context, s OrderStatus: dbtypes.OrderStatusAbandoned, OrderStatusMessage: sql.NullString{String: "reorged", Valid: true}, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return false, fmt.Errorf("failed to set fill status to abandoned: %w", err) } lmt.Logger(ctx).Info("abandoning transaction due to reorg", zap.String("orderId", order.OrderID), zap.String("sourceChainID", order.SourceChainID)) @@ -382,6 +390,7 @@ func (r *orderFulfillmentHandler) InitiateTimeout(ctx context.Context, order db. OrderID: sql.NullInt64{Int64: order.ID, Valid: true}, TxType: dbtypes.TxTypeInitiateTimeout, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return fmt.Errorf("failed to get submitted txs: %w", err) } else if len(submittedTxs) > 0 { return nil @@ -401,6 +410,7 @@ func (r *orderFulfillmentHandler) InitiateTimeout(ctx context.Context, order db. TxType: dbtypes.TxTypeInitiateTimeout, TxStatus: dbtypes.TxStatusPending, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) return fmt.Errorf("failed to insert raw tx %w", err) } return nil diff --git a/orderfulfiller/orderfulfiller.go b/orderfulfiller/orderfulfiller.go index 43fde48..464ad3c 100644 --- a/orderfulfiller/orderfulfiller.go +++ b/orderfulfiller/orderfulfiller.go @@ -12,6 +12,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" ) const ( @@ -82,6 +83,7 @@ func (r *OrderFulfiller) dispatchOrderFills(ctx context.Context) { case <-ticker.C: orders, err := r.db.GetAllOrdersWithOrderStatus(ctx, dbtypes.OrderStatusPending) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) lmt.Logger(ctx).Error("error getting pending orders", zap.Error(err)) continue } @@ -103,6 +105,7 @@ func (r *OrderFulfiller) startOrderTimeoutWorker(ctx context.Context) { case <-ticker.C: orders, err := r.db.GetAllOrdersWithOrderStatus(ctx, dbtypes.OrderStatusExpiredPendingRefund) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) lmt.Logger(ctx).Error("error getting expired orders", zap.Error(err)) continue } diff --git a/ordersettler/ordersettler.go b/ordersettler/ordersettler.go index e5f4086..1229106 100644 --- a/ordersettler/ordersettler.go +++ b/ordersettler/ordersettler.go @@ -155,6 +155,7 @@ func (r *OrderSettler) findNewSettlements(ctx context.Context) error { SourceChainGatewayContractAddress: sourceGatewayAddress, }) if err != nil && !errors.Is(err, sql.ErrNoRows) { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return fmt.Errorf("order %s does not exist, setting status to abandoned: %w", fill.OrderID, err) } continue @@ -170,6 +171,7 @@ func (r *OrderSettler) findNewSettlements(ctx context.Context) error { metrics.FromContext(ctx).IncOrderSettlements(sourceChainID, chain.ChainID, dbtypes.SettlementStatusPending) if err != nil && !errors.Is(err, sql.ErrNoRows) { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) return fmt.Errorf("failed to insert settlement: %w", err) } } @@ -255,6 +257,7 @@ func (r *OrderSettler) verifyOrderSettlements(ctx context.Context) error { func (r *OrderSettler) PendingSettlementBatches(ctx context.Context) ([]types.SettlementBatch, error) { pendingSettlements, err := r.db.GetAllOrderSettlementsWithSettlementStatus(ctx, dbtypes.SettlementStatusPending) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return nil, fmt.Errorf("getting orders pending settlement: %w", err) } return types.IntoSettlementBatches(pendingSettlements) @@ -263,6 +266,7 @@ func (r *OrderSettler) PendingSettlementBatches(ctx context.Context) ([]types.Se func (r *OrderSettler) InitiatedSettlements(ctx context.Context) ([]db.OrderSettlement, error) { iniatedSettlements, err := r.db.GetAllOrderSettlementsWithSettlementStatus(ctx, dbtypes.SettlementStatusSettlementInitiated) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return nil, fmt.Errorf("getting orders that have initiated settlement: %w", err) } return iniatedSettlements, nil @@ -356,6 +360,7 @@ func (r *OrderSettler) SettleBatch(ctx context.Context, batch types.SettlementBa return nil }, nil) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) return fmt.Errorf("recording batch settlement result: %w", err) } @@ -394,6 +399,7 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db. SettlementStatus: dbtypes.SettlementStatusFailed, SettlementStatusMessage: sql.NullString{String: failure.String(), Valid: true}, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return fmt.Errorf("failed to set relay status to failed: %w", err) } if gasCost == nil { @@ -408,6 +414,7 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db. SourceChainGatewayContractAddress: settlement.SourceChainGatewayContractAddress, SettlementStatus: dbtypes.SettlementStatusSettlementInitiated, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return fmt.Errorf("failed to set relay status to complete: %w", err) } } @@ -425,6 +432,7 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db. SourceChainGatewayContractAddress: settlement.SourceChainGatewayContractAddress, SettlementStatus: dbtypes.SettlementStatusComplete, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return fmt.Errorf("failed to set relay status to complete: %w", err) } diff --git a/shared/metrics/metrics.go b/shared/metrics/metrics.go index 12ab807..d5a40ab 100644 --- a/shared/metrics/metrics.go +++ b/shared/metrics/metrics.go @@ -18,6 +18,7 @@ const ( orderStatusLabel = "order_status" transferStatusLabel = "transfer_status" settlementStatusLabel = "settlement_status" + operationLabel = "operation" ) type Metrics interface { @@ -42,6 +43,8 @@ type Metrics interface { ObserveTransferSizeExceeded(sourceChainID, destinationChainID string, amountExceededBy uint64) ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBpsExceededBy int64) + + IncDatabaseErrors(operation string) } type metricsContextKey struct{} @@ -79,6 +82,8 @@ type PromMetrics struct { transferSizeExceeded metrics.Histogram feeBpsRejections metrics.Histogram + + databaseErrors metrics.Counter } func NewPromMetrics() Metrics { @@ -156,6 +161,11 @@ func NewPromMetrics() Metrics { Help: "histogram of fee bps that were rejected for being too low", Buckets: []float64{1, 5, 10, 25, 50, 100, 200, 500, 1000}, }, []string{sourceChainIDLabel, destinationChainIDLabel}), + databaseErrors: prom.NewCounterFrom(stdprom.CounterOpts{ + Namespace: "solver", + Name: "database_errors_total", + Help: "number of errors encountered when making database calls", + }, []string{}), } } @@ -227,6 +237,10 @@ func (m *PromMetrics) ObserveFeeBpsRejection(sourceChainID, destinationChainID s ).Observe(float64(feeBps)) } +func (m *PromMetrics) IncDatabaseErrors(operation string) { + m.databaseErrors.With(operationLabel, operation).Add(1) +} + type NoOpMetrics struct{} func (n NoOpMetrics) IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) { @@ -254,6 +268,7 @@ func (n NoOpMetrics) IncHyperlaneMessages(sourceChainID, destinationChainID, mes func (n NoOpMetrics) DecHyperlaneMessages(sourceChainID, destinationChainID, messageStatus string) {} func (n NoOpMetrics) ObserveTransferSizeExceeded(sourceChainID, destinationChainID string, transferSize uint64) { } +func (n NoOpMetrics) IncDatabaseErrors(operation string) {} func (n NoOpMetrics) ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBps int64) {} func NewNoOpMetrics() Metrics { return &NoOpMetrics{} diff --git a/transfermonitor/transfermonitor.go b/transfermonitor/transfermonitor.go index 1884f2b..c191548 100644 --- a/transfermonitor/transfermonitor.go +++ b/transfermonitor/transfermonitor.go @@ -6,12 +6,13 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "github.com/skip-mev/go-fast-solver/shared/metrics" "math/big" "strings" "sync" "time" + "github.com/skip-mev/go-fast-solver/shared/metrics" + "cosmossdk.io/math" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -81,6 +82,7 @@ func (t *TransferMonitor) Start(ctx context.Context) error { var startBlockHeight uint64 transferMonitorMetadata, err := t.db.GetTransferMonitorMetadata(ctx, chainID) if err != nil && !strings.Contains(err.Error(), "no rows in result set") { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) lmt.Logger(ctx).Error("Error getting transfer monitor metadata", zap.Error(err)) continue } else if err == nil { @@ -139,6 +141,7 @@ func (t *TransferMonitor) Start(ctx context.Context) error { _, err := t.db.InsertOrder(ctx, toInsert) if err != nil && !strings.Contains(err.Error(), "sql: no rows in result set") { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) lmt.Logger(ctx).Error("Error inserting order", zap.Error(err)) errorInsertingOrder = true break @@ -156,6 +159,7 @@ func (t *TransferMonitor) Start(ctx context.Context) error { HeightLastSeen: int64(endBlockHeight), }) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) lmt.Logger(ctx).Error("Error inserting transfer monitor metadata", zap.Error(err)) continue } diff --git a/txverifier/txverifier.go b/txverifier/txverifier.go index 558a0f6..096e459 100644 --- a/txverifier/txverifier.go +++ b/txverifier/txverifier.go @@ -4,10 +4,11 @@ import ( "context" "database/sql" "fmt" + "time" + dbtypes "github.com/skip-mev/go-fast-solver/db" "github.com/skip-mev/go-fast-solver/shared/clientmanager" "github.com/skip-mev/go-fast-solver/shared/metrics" - "time" coingecko2 "github.com/skip-mev/go-fast-solver/shared/clients/coingecko" @@ -64,6 +65,7 @@ func (r *TxVerifier) Run(ctx context.Context) { func (r *TxVerifier) verifyTxs(ctx context.Context) { submittedTxs, err := r.db.GetSubmittedTxsWithStatus(ctx, dbtypes.TxStatusPending) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) lmt.Logger(ctx).Error("error getting pending txs", zap.Error(err)) return } @@ -110,6 +112,7 @@ func (r *TxVerifier) VerifyTx(ctx context.Context, submittedTx db.SubmittedTx) e ChainID: submittedTx.ChainID, TxStatusMessage: sql.NullString{String: failure.String(), Valid: true}, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return fmt.Errorf("failed to set tx status to failed: %w", err) } return fmt.Errorf("tx failed: %s", failure.String()) @@ -120,6 +123,7 @@ func (r *TxVerifier) VerifyTx(ctx context.Context, submittedTx db.SubmittedTx) e TxHash: submittedTx.TxHash, ChainID: submittedTx.ChainID, }); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return fmt.Errorf("failed to set tx status to success: %w", err) } } From 3519d4208db17553f38d0e198441062248aaf633 Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Mon, 28 Oct 2024 15:37:15 -0400 Subject: [PATCH 04/11] cleanup --- fundrebalancer/fundrebalancer.go | 4 ++-- fundrebalancer/transfermonitor.go | 2 +- hyperlane/relayer.go | 2 +- hyperlane/relayer_runner.go | 3 +-- transfermonitor/transfermonitor.go | 3 +-- txverifier/txverifier.go | 14 ++++++-------- 6 files changed, 12 insertions(+), 16 deletions(-) diff --git a/fundrebalancer/fundrebalancer.go b/fundrebalancer/fundrebalancer.go index cbf4f42..00aa56b 100644 --- a/fundrebalancer/fundrebalancer.go +++ b/fundrebalancer/fundrebalancer.go @@ -4,17 +4,17 @@ import ( "encoding/hex" "encoding/json" "fmt" - dbtypes "github.com/skip-mev/go-fast-solver/db" - "github.com/skip-mev/go-fast-solver/shared/metrics" "math/big" "os" "time" + dbtypes "github.com/skip-mev/go-fast-solver/db" "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/shared/clients/skipgo" "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/evmrpc" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "github.com/skip-mev/go-fast-solver/shared/signing" "github.com/skip-mev/go-fast-solver/shared/signing/evm" "go.uber.org/zap" diff --git a/fundrebalancer/transfermonitor.go b/fundrebalancer/transfermonitor.go index f863060..a183a70 100644 --- a/fundrebalancer/transfermonitor.go +++ b/fundrebalancer/transfermonitor.go @@ -3,13 +3,13 @@ package fundrebalancer import ( "context" "fmt" - "github.com/skip-mev/go-fast-solver/shared/metrics" "time" "github.com/skip-mev/go-fast-solver/db" genDB "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/shared/clients/skipgo" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "go.uber.org/zap" ) diff --git a/hyperlane/relayer.go b/hyperlane/relayer.go index ae541f0..ed27ee5 100644 --- a/hyperlane/relayer.go +++ b/hyperlane/relayer.go @@ -4,7 +4,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/skip-mev/go-fast-solver/shared/metrics" "strings" @@ -13,6 +12,7 @@ import ( "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "go.uber.org/zap" "golang.org/x/net/context" ) diff --git a/hyperlane/relayer_runner.go b/hyperlane/relayer_runner.go index 65a731f..e97c822 100644 --- a/hyperlane/relayer_runner.go +++ b/hyperlane/relayer_runner.go @@ -7,12 +7,11 @@ import ( "fmt" "time" - "github.com/skip-mev/go-fast-solver/shared/metrics" - dbtypes "github.com/skip-mev/go-fast-solver/db" "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "go.uber.org/zap" ) diff --git a/transfermonitor/transfermonitor.go b/transfermonitor/transfermonitor.go index c191548..c51f4b2 100644 --- a/transfermonitor/transfermonitor.go +++ b/transfermonitor/transfermonitor.go @@ -11,8 +11,6 @@ import ( "sync" "time" - "github.com/skip-mev/go-fast-solver/shared/metrics" - "cosmossdk.io/math" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -23,6 +21,7 @@ import ( "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/contracts/fast_transfer_gateway" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" "github.com/skip-mev/go-fast-solver/shared/tmrpc" "go.uber.org/zap" "golang.org/x/sync/errgroup" diff --git a/txverifier/txverifier.go b/txverifier/txverifier.go index 096e459..56d2d43 100644 --- a/txverifier/txverifier.go +++ b/txverifier/txverifier.go @@ -6,18 +6,16 @@ import ( "fmt" "time" - dbtypes "github.com/skip-mev/go-fast-solver/db" - "github.com/skip-mev/go-fast-solver/shared/clientmanager" - "github.com/skip-mev/go-fast-solver/shared/metrics" - - coingecko2 "github.com/skip-mev/go-fast-solver/shared/clients/coingecko" - "go.uber.org/zap" "golang.org/x/sync/errgroup" + dbtypes "github.com/skip-mev/go-fast-solver/db" "github.com/skip-mev/go-fast-solver/db/gen/db" + "github.com/skip-mev/go-fast-solver/shared/clientmanager" + "github.com/skip-mev/go-fast-solver/shared/clients/coingecko" "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" ) type Config struct { @@ -36,7 +34,7 @@ type Database interface { type TxVerifier struct { db Database clientManager *clientmanager.ClientManager - PriceClient coingecko2.PriceClient + PriceClient coingecko.PriceClient } func NewTxVerifier(ctx context.Context, db Database, clientManager *clientmanager.ClientManager) (*TxVerifier, error) { @@ -45,7 +43,7 @@ func NewTxVerifier(ctx context.Context, db Database, clientManager *clientmanage return &TxVerifier{ db: db, clientManager: clientManager, - PriceClient: coingecko2.NewCachedPriceClient(coingecko2.DefaultCoingeckoClient(coingeckoConfig), coingeckoConfig.CacheRefreshInterval), + PriceClient: coingecko.NewCachedPriceClient(coingecko.DefaultCoingeckoClient(coingeckoConfig), coingeckoConfig.CacheRefreshInterval), }, nil } From 75ae93d4dd39bb8d3b44e67000e8161924dbd917 Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Mon, 28 Oct 2024 15:51:32 -0400 Subject: [PATCH 05/11] change latency unit --- fundrebalancer/fundrebalancer.go | 6 +++--- shared/metrics/metrics.go | 32 ++++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/fundrebalancer/fundrebalancer.go b/fundrebalancer/fundrebalancer.go index 00aa56b..b779c13 100644 --- a/fundrebalancer/fundrebalancer.go +++ b/fundrebalancer/fundrebalancer.go @@ -4,17 +4,17 @@ import ( "encoding/hex" "encoding/json" "fmt" + dbtypes "github.com/skip-mev/go-fast-solver/db" + "github.com/skip-mev/go-fast-solver/shared/metrics" "math/big" "os" "time" - dbtypes "github.com/skip-mev/go-fast-solver/db" "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/shared/clients/skipgo" "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/evmrpc" "github.com/skip-mev/go-fast-solver/shared/lmt" - "github.com/skip-mev/go-fast-solver/shared/metrics" "github.com/skip-mev/go-fast-solver/shared/signing" "github.com/skip-mev/go-fast-solver/shared/signing/evm" "go.uber.org/zap" @@ -255,6 +255,7 @@ func (r *FundRebalancer) MoveFundsToChain( if err != nil { return nil, nil, fmt.Errorf("submitting signed txns required for fund rebalancing: %w", err) } + metrics.FromContext(ctx).IncFundsRebalanceTransfers(rebalanceFromChainID, rebalanceToChain, dbtypes.RebalanceTransactionStatusPending) totalUSDCcMoved = new(big.Int).Add(totalUSDCcMoved, usdcToRebalance) hashes = append(hashes, txnHashes...) @@ -264,7 +265,6 @@ func (r *FundRebalancer) MoveFundsToChain( if remainingUSDCNeeded.Cmp(big.NewInt(0)) <= 0 { return hashes, totalUSDCcMoved, nil } - metrics.FromContext(ctx).IncFundsRebalanceTransfers(rebalanceFromChainID, rebalanceToChain, dbtypes.RebalanceTransactionStatusPending) } // we have moved all available funds from all available chains diff --git a/shared/metrics/metrics.go b/shared/metrics/metrics.go index d5a40ab..9ac83a4 100644 --- a/shared/metrics/metrics.go +++ b/shared/metrics/metrics.go @@ -80,8 +80,8 @@ type PromMetrics struct { hplCheckpointingErrors metrics.Counter hplLatency metrics.Histogram - transferSizeExceeded metrics.Histogram - feeBpsRejections metrics.Histogram + amountTransferSizeExceeded metrics.Histogram + feeBpsRejections metrics.Histogram databaseErrors metrics.Counter } @@ -116,15 +116,15 @@ func NewPromMetrics() Metrics { }, []string{successLabel, chainIDLabel}), fillLatency: prom.NewHistogramFrom(stdprom.HistogramOpts{ Namespace: "solver", - Name: "latency_per_fill", - Help: "latency from source transaction to fill completion, paginated by source and destination chain id", - Buckets: []float64{30, 60, 300, 600, 900, 1200, 1500, 1800, 2400, 3000, 3600}, + Name: "latency_per_fill_minutes", + Help: "latency from source transaction to fill completion, paginated by source and destination chain id (in minutes)", + Buckets: []float64{5, 15, 30, 60, 120, 180}, }, []string{sourceChainIDLabel, destinationChainIDLabel, orderStatusLabel}), settlementLatency: prom.NewHistogramFrom(stdprom.HistogramOpts{ Namespace: "solver", - Name: "latency_per_settlement", - Help: "latency from source transaction to fill completion, paginated by source and destination chain id", - Buckets: []float64{30, 60, 300, 600, 900, 1200, 1500, 1800, 2400, 3000, 3600}, + Name: "latency_per_settlement_minutes", + Help: "latency from source transaction to fill completion, paginated by source and destination chain id (in minutes)", + Buckets: []float64{5, 15, 30, 60, 120, 180}, }, []string{sourceChainIDLabel, destinationChainIDLabel, settlementStatusLabel}), hplMessages: prom.NewGaugeFrom(stdprom.GaugeOpts{ Namespace: "solver", @@ -139,14 +139,14 @@ func NewPromMetrics() Metrics { }, []string{}), hplLatency: prom.NewHistogramFrom(stdprom.HistogramOpts{ Namespace: "solver", - Name: "latency_per_hpl_message", - Help: "latency for hyperlane message relaying, paginated by status, source and destination chain id", + Name: "latency_per_hpl_message_seconds", + Help: "latency for hyperlane message relaying, paginated by status, source and destination chain id (in seconds)", Buckets: []float64{30, 60, 300, 600, 900, 1200, 1500, 1800, 2400, 3000, 3600}, }, []string{sourceChainIDLabel, destinationChainIDLabel, transferStatusLabel}), - transferSizeExceeded: prom.NewHistogramFrom(stdprom.HistogramOpts{ + amountTransferSizeExceeded: prom.NewHistogramFrom(stdprom.HistogramOpts{ Namespace: "solver", - Name: "transfer_size_exceeded", - Help: "histogram of transfer sizes that exceeded max fill size", + Name: "amount_transfer_size_exceeded", + Help: "histogram of fill orders that exceeded max fill size", Buckets: []float64{ 100000000, // 100 USDC 1000000000, // 1,000 USDC @@ -178,11 +178,11 @@ func (m *PromMetrics) IncTransactionVerified(success bool, chainID string) { } func (m *PromMetrics) ObserveFillLatency(sourceChainID, destinationChainID, orderStatus string, latency time.Duration) { - m.fillLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, orderStatusLabel, orderStatus).Observe(latency.Seconds()) + m.fillLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, orderStatusLabel, orderStatus).Observe(latency.Minutes()) } func (m *PromMetrics) ObserveSettlementLatency(sourceChainID, destinationChainID, settlementStatus string, latency time.Duration) { - m.settlementLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, settlementStatusLabel, settlementStatus).Observe(latency.Seconds()) + m.settlementLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, settlementStatusLabel, settlementStatus).Observe(latency.Minutes()) } func (m *PromMetrics) ObserveHyperlaneLatency(sourceChainID, destinationChainID, transferStatus string, latency time.Duration) { @@ -224,7 +224,7 @@ func (m *PromMetrics) DecHyperlaneMessages(sourceChainID, destinationChainID, me } func (m *PromMetrics) ObserveTransferSizeExceeded(sourceChainID, destinationChainID string, transferSize uint64) { - m.transferSizeExceeded.With( + m.amountTransferSizeExceeded.With( sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, ).Observe(float64(transferSize)) From 65abe03ea7072599eb7be73b192f5ae1b42308b8 Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Mon, 28 Oct 2024 15:57:25 -0400 Subject: [PATCH 06/11] add missing db inc errors calls --- fundrebalancer/fundrebalancer.go | 7 +++++-- fundrebalancer/transfermonitor.go | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/fundrebalancer/fundrebalancer.go b/fundrebalancer/fundrebalancer.go index b779c13..cbafabf 100644 --- a/fundrebalancer/fundrebalancer.go +++ b/fundrebalancer/fundrebalancer.go @@ -4,12 +4,13 @@ import ( "encoding/hex" "encoding/json" "fmt" - dbtypes "github.com/skip-mev/go-fast-solver/db" - "github.com/skip-mev/go-fast-solver/shared/metrics" "math/big" "os" "time" + dbtypes "github.com/skip-mev/go-fast-solver/db" + "github.com/skip-mev/go-fast-solver/shared/metrics" + "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/shared/clients/skipgo" "github.com/skip-mev/go-fast-solver/shared/config" @@ -341,6 +342,7 @@ func (r *FundRebalancer) usdcBalance(ctx context.Context, chainID string) (*big. func (r *FundRebalancer) pendingUSDCBalance(ctx context.Context, chainID string) (*big.Int, error) { pendingTransfers, err := r.database.GetPendingRebalanceTransfersToChain(ctx, chainID) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return nil, fmt.Errorf("getting pending rebalance transfers to chain from db: %w", err) } @@ -591,6 +593,7 @@ func (r *FundRebalancer) SubmitTxns( Amount: signedTxn.Amount.String(), } if _, err := r.database.InsertRebalanceTransfer(ctx, args); err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) return nil, fmt.Errorf("inserting rebalance transaction with hash %s into db: %w", hash, err) } diff --git a/fundrebalancer/transfermonitor.go b/fundrebalancer/transfermonitor.go index a183a70..9cfc98c 100644 --- a/fundrebalancer/transfermonitor.go +++ b/fundrebalancer/transfermonitor.go @@ -6,6 +6,7 @@ import ( "time" "github.com/skip-mev/go-fast-solver/db" + dbtypes "github.com/skip-mev/go-fast-solver/db" genDB "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/shared/clients/skipgo" "github.com/skip-mev/go-fast-solver/shared/lmt" @@ -54,6 +55,7 @@ func (t *TransferTracker) TrackPendingTransfers(ctx context.Context) { func (t *TransferTracker) UpdateTransfers(ctx context.Context) error { pendingTransfers, err := t.database.GetAllPendingRebalanceTransfers(ctx) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.GET) return fmt.Errorf("getting all pending rebalance transfers: %w", err) } @@ -121,13 +123,14 @@ func (t *TransferTracker) updateTransferStatus(ctx context.Context, transferID i zap.String("destinationChainID", destinationChainID), zap.String("error", transferError), ) + metrics.FromContext(ctx).IncFundsRebalanceTransfers(sourceChainID, destinationChainID, db.RebalanceTransactionStatusFailed) err = t.database.UpdateTransferStatus(ctx, genDB.UpdateTransferStatusParams{ Status: db.RebalanceTransactionStatusFailed, ID: transferID, }) if err != nil { - metrics.FromContext(ctx).IncFundsRebalanceTransfers(sourceChainID, destinationChainID, db.RebalanceTransactionStatusFailed) + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return fmt.Errorf("updating transfer status to failed for hash %s on chain %s: %w", hash, sourceChainID, err) } @@ -140,13 +143,14 @@ func (t *TransferTracker) updateTransferStatus(ctx context.Context, transferID i zap.String("sourceChainID", sourceChainID), zap.String("destinationChainID", destinationChainID), ) - metrics.FromContext(ctx).IncFundsRebalanceTransfers(sourceChainID, destinationChainID, db.RebalanceTransactionStatusSuccess) + err = t.database.UpdateTransferStatus(ctx, genDB.UpdateTransferStatusParams{ Status: db.RebalanceTransactionStatusSuccess, ID: transferID, }) if err != nil { + metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.UPDATE) return fmt.Errorf("updating transfer status to completed for hash %s on chain %s: %w", hash, sourceChainID, err) } From fb8d356045ce620c457b4ad99ab1ba8991714b26 Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Mon, 28 Oct 2024 16:10:13 -0400 Subject: [PATCH 07/11] more cleanup --- hyperlane/relayer_runner.go | 2 +- .../order_fulfillment_handler/order_fulfillment_handler.go | 2 +- ordersettler/ordersettler.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hyperlane/relayer_runner.go b/hyperlane/relayer_runner.go index e97c822..d06c0ed 100644 --- a/hyperlane/relayer_runner.go +++ b/hyperlane/relayer_runner.go @@ -129,7 +129,7 @@ func (r *RelayerRunner) checkHyperlaneTransferStatus(ctx context.Context, transf } if delivered { metrics.FromContext(ctx).IncHyperlaneMessages(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusSuccess) - metrics.FromContext(ctx).DecHyperlaneMessages(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusSuccess) + metrics.FromContext(ctx).DecHyperlaneMessages(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusPending) metrics.FromContext(ctx).ObserveHyperlaneLatency(transfer.SourceChainID, transfer.DestinationChainID, dbtypes.TransferStatusSuccess, time.Since(transfer.CreatedAt)) if _, err := r.db.SetMessageStatus(ctx, db.SetMessageStatusParams{ diff --git a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go index c3708cf..e3d749b 100644 --- a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go +++ b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go @@ -197,10 +197,10 @@ func (r *orderFulfillmentHandler) FillOrder( } txHash, rawTx, _, err := destinationChainBridgeClient.FillOrder(ctx, order, destinationChainGatewayContractAddress) + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) if err != nil { return "", fmt.Errorf("filling order on destination chain at address %s: %w", destinationChainBridgeClient, err) } - metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) if _, err := r.db.InsertSubmittedTx(ctx, db.InsertSubmittedTxParams{ OrderID: sql.NullInt64{Int64: order.ID, Valid: true}, diff --git a/ordersettler/ordersettler.go b/ordersettler/ordersettler.go index 1229106..0e0bf51 100644 --- a/ordersettler/ordersettler.go +++ b/ordersettler/ordersettler.go @@ -168,12 +168,12 @@ func (r *OrderSettler) findNewSettlements(ctx context.Context) error { OrderID: fill.OrderID, SettlementStatus: dbtypes.SettlementStatusPending, }) - metrics.FromContext(ctx).IncOrderSettlements(sourceChainID, chain.ChainID, dbtypes.SettlementStatusPending) if err != nil && !errors.Is(err, sql.ErrNoRows) { metrics.FromContext(ctx).IncDatabaseErrors(dbtypes.INSERT) return fmt.Errorf("failed to insert settlement: %w", err) } + metrics.FromContext(ctx).IncOrderSettlements(sourceChainID, chain.ChainID, dbtypes.SettlementStatusPending) } } return nil From 0e5111011e0ccd8a040c6d8118500c3129a78441 Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Thu, 31 Oct 2024 01:36:57 -0400 Subject: [PATCH 08/11] feat: add gas balance metric --- cmd/solver/main.go | 4 +- config/local/config.yml | 36 +++--- fundrebalancer/fundrebalancer.go | 19 +++ hyperlane/relayer.go | 20 ++- .../order_fulfillment_handler.go | 33 ++++- ordersettler/ordersettler.go | 15 +++ shared/bridges/cctp/bridge_client.go | 2 +- shared/bridges/cctp/cosmos_bridge_client.go | 14 +- shared/bridges/cctp/evm_bridge_client.go | 2 +- shared/bridges/cctp/svm_bridge_client.go | 4 +- shared/config/config.go | 121 +++++++++++++----- shared/metrics/metrics.go | 43 ++++++- shared/utils/util.go | 37 ++++++ 13 files changed, 286 insertions(+), 64 deletions(-) create mode 100644 shared/utils/util.go diff --git a/cmd/solver/main.go b/cmd/solver/main.go index d991a0b..05c2f1f 100644 --- a/cmd/solver/main.go +++ b/cmd/solver/main.go @@ -134,7 +134,7 @@ func main() { }) eg.Go(func() error { - r, err := fundrebalancer.NewFundRebalancer(ctx, *keysPath, skipgo, evmManager, db.New(dbConn)) + r, err := fundrebalancer.NewFundRebalancer(ctx, *keysPath, skipgo, evmManager, clientManager, db.New(dbConn)) if err != nil { return fmt.Errorf("creating fund rebalancer: %w", err) } @@ -148,7 +148,7 @@ func main() { return fmt.Errorf("creating hyperlane multi client from config: %w", err) } - relayer := hyperlane.NewRelayer(hype, make(map[string]string)) + relayer := hyperlane.NewRelayer(hype, make(map[string]string), clientManager) err = hyperlane.NewRelayerRunner(db.New(dbConn), hype, relayer).Run(ctx) if err != nil { return fmt.Errorf("relaying message: %v", err) diff --git a/config/local/config.yml b/config/local/config.yml index 0eed215..6ad6e28 100644 --- a/config/local/config.yml +++ b/config/local/config.yml @@ -23,7 +23,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48 - signer_gas_balance: + gas_balance: warning_threshold_wei: 4290000000000000000 critical_threshold_wei: 1430000000000000000 ethereum-testnet: @@ -38,7 +38,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x1c7D4B196Cb0C7B01d743Fbc6116a902379C7238 - signer_gas_balance: + gas_balance: warning_threshold_wei: 250000000000000000 critical_threshold_wei: 0 avalanche: @@ -55,7 +55,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0xB97EF9Ef8734C71904D8002F8b6Bc66Dd9c48a6E - signer_gas_balance: + gas_balance: warning_threshold_wei: 1720000000000000000 critical_threshold_wei: 580000000000000000 avalanche-testnet: @@ -70,7 +70,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x5425890298aed601595a70ab815c96711a31bc65 - signer_gas_balance: + gas_balance: warning_threshold_wei: 1000000000000000000 critical_threshold_wei: 0 optimism: @@ -86,7 +86,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x0b2c639c533813f4aa9d7837caf62653d097ff85 - signer_gas_balance: + gas_balance: warning_threshold_wei: 180000000000000000 critical_threshold_wei: 60000000000000000 optimism-testnet: @@ -101,7 +101,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x5fd84259d66Cd46123540766Be93DFE6D43130D7 - signer_gas_balance: + gas_balance: warning_threshold_wei: 250000000000000000 critical_threshold_wei: 0 arbitrum: @@ -122,7 +122,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0xaf88d065e77c8cC2239327C5EDb3A432268e5831 - signer_gas_balance: + gas_balance: warning_threshold_wei: 180000000000000000 critical_threshold_wei: 60000000000000000 relayer: @@ -141,7 +141,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x75faf114eafb1BDbe2F0316DF893fd58CE46AA4d - signer_gas_balance: + gas_balance: warning_threshold_wei: 250000000000000000 critical_threshold_wei: 0 osmosis: @@ -164,7 +164,7 @@ chains: grpc: grpc_tls_enabled: false usdc_denom: "ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4" - signer_gas_balance: + gas_balance: warning_threshold_wei: 42860000 critical_threshold_wei: 14290000 gas_price: 0.0025 @@ -185,7 +185,7 @@ chains: address_prefix: "neutron" rpc: rpc_basic_auth_var: - signer_gas_balance: + gas_balance: warning_threshold_wei: 42860000 critical_threshold_wei: 14290000 noble: @@ -200,7 +200,7 @@ chains: address_prefix: "noble" rpc: rpc_basic_auth_var: - signer_gas_balance: + gas_balance: warning_threshold_wei: 42860000 critical_threshold_wei: 14290000 noble-testnet: @@ -213,7 +213,7 @@ chains: cosmos: address_prefix: "noble" rpc: - signer_gas_balance: + gas_balance: warning_threshold_wei: 0 critical_threshold_wei: 0 base-mainnet: @@ -229,7 +229,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913 - signer_gas_balance: + gas_balance: warning_threshold_wei: 180000000000000000 critical_threshold_wei: 60000000000000000 base-testnet: @@ -244,7 +244,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x036CbD53842c5426634e7929541eC2318f3dCF7e - signer_gas_balance: + gas_balance: warning_threshold_wei: 250000000000000000 critical_threshold_wei: 0 polygon: @@ -260,7 +260,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x3c499c542cef5e3811e1192ce70d8cc03d5c3359 - signer_gas_balance: + gas_balance: warning_threshold_wei: 15000000000000000000 critical_threshold_wei: 5000000000000000000 min_gas_tip_cap: 30000000000 @@ -276,7 +276,7 @@ chains: rpc_basic_auth_var: contracts: usdc_erc20_address: 0x41e94eb019c0762f9bfcf9fb1e58725bfb0e7582 - signer_gas_balance: + gas_balance: warning_threshold_wei: 250000000000000000 critical_threshold_wei: 0 solana: @@ -296,7 +296,7 @@ chains: message_transmitter: CCTPmbSD7gX1bxKPAmg77w8oFzNFpaQiQUWD43TKaecd token_messenger_minter: CCTPiPYPc6AsJuwueEnWgSgucamXDZwBd53dQ11YiKX3 usdc: EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v - signer_gas_balance: + gas_balance: warning_threshold_wei: 2150000000 critical_threshold_wei: 720000000 priority_fee: 600000 @@ -314,6 +314,6 @@ chains: message_transmitter: CCTPmbSD7gX1bxKPAmg77w8oFzNFpaQiQUWD43TKaecd token_messenger_minter: CCTPiPYPc6AsJuwueEnWgSgucamXDZwBd53dQ11YiKX3 usdc: 4zMMC9srt5Ri5X14GAgXhaHii3GnPAEERYPJgZJDncDU - signer_gas_balance: + gas_balance: warning_threshold_wei: 1000000000 critical_threshold_wei: 0 diff --git a/fundrebalancer/fundrebalancer.go b/fundrebalancer/fundrebalancer.go index cbafabf..28ae3f3 100644 --- a/fundrebalancer/fundrebalancer.go +++ b/fundrebalancer/fundrebalancer.go @@ -4,6 +4,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/skip-mev/go-fast-solver/shared/clientmanager" + "github.com/skip-mev/go-fast-solver/shared/utils" "math/big" "os" "time" @@ -38,6 +40,7 @@ type FundRebalancer struct { chainIDToPrivateKey map[string]string skipgo skipgo.SkipGoClient evmClientManager evmrpc.EVMRPCClientManager + clientManager *clientmanager.ClientManager config map[string]config.FundRebalancerConfig database Database trasferTracker *TransferTracker @@ -48,6 +51,7 @@ func NewFundRebalancer( keysPath string, skipgo skipgo.SkipGoClient, evmClientManager evmrpc.EVMRPCClientManager, + clientmanager *clientmanager.ClientManager, database Database, ) (*FundRebalancer, error) { chainIDToPriavateKey, err := loadChainIDToPrivateKeyMap(keysPath) @@ -59,6 +63,7 @@ func NewFundRebalancer( chainIDToPrivateKey: chainIDToPriavateKey, skipgo: skipgo, evmClientManager: evmClientManager, + clientManager: clientmanager, config: config.GetConfigReader(ctx).Config().FundRebalancer, database: database, trasferTracker: NewTransferTracker(skipgo, database), @@ -256,6 +261,20 @@ func (r *FundRebalancer) MoveFundsToChain( if err != nil { return nil, nil, fmt.Errorf("submitting signed txns required for fund rebalancing: %w", err) } + + sourceChainClient, err := r.clientManager.GetClient(ctx, rebalanceFromChainID) + if err != nil { + lmt.Logger(ctx).Error("failed to get chain client to monitor gas balance", zap.Error(err)) + } + + // dont fail if we cant get the chain client, just log an error + if sourceChainClient != nil { + err = utils.MonitorGasBalance(ctx, rebalanceFromChainID, sourceChainClient) + if err != nil { + lmt.Logger(ctx).Error("failed to monitor gas balance", zap.Error(err), zap.String("chainID", rebalanceFromChainID)) + } + } + metrics.FromContext(ctx).IncFundsRebalanceTransfers(rebalanceFromChainID, rebalanceToChain, dbtypes.RebalanceTransactionStatusPending) totalUSDCcMoved = new(big.Int).Add(totalUSDCcMoved, usdcToRebalance) diff --git a/hyperlane/relayer.go b/hyperlane/relayer.go index ed27ee5..0a8ba5a 100644 --- a/hyperlane/relayer.go +++ b/hyperlane/relayer.go @@ -4,6 +4,8 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/skip-mev/go-fast-solver/shared/clientmanager" + "github.com/skip-mev/go-fast-solver/shared/utils" "strings" @@ -24,12 +26,14 @@ type Relayer interface { type relayer struct { hyperlane Client storageLocationOverrides map[string]string + clientManager *clientmanager.ClientManager } -func NewRelayer(hyperlaneClient Client, storageLocationOverrides map[string]string) Relayer { +func NewRelayer(hyperlaneClient Client, storageLocationOverrides map[string]string, clientManager *clientmanager.ClientManager) Relayer { return &relayer{ hyperlane: hyperlaneClient, storageLocationOverrides: storageLocationOverrides, + clientManager: clientManager, } } @@ -141,6 +145,20 @@ func (r *relayer) Relay(ctx context.Context, originChainID string, initiateTxHas return "", "", fmt.Errorf("getting destination chain config for chainID %s: %w", destinationChainID, err) } + originChainClient, err := r.clientManager.GetClient(ctx, originChainID) + if err != nil { + lmt.Logger(ctx).Error("failed to get chain client to monitor gas balance", + zap.Error(err), zap.String("chainID", originChainID)) + } + + // dont fail if we cant get the chain client, just log an error + if originChainClient != nil { + err = utils.MonitorGasBalance(ctx, originChainID, originChainClient) + if err != nil { + lmt.Logger(ctx).Error("failed to monitor gas balance", zap.Error(err), zap.String("chainID", originChainID)) + } + } + lmt.Logger(ctx).Info( fmt.Sprintf("relayed hyperlane message from %s to %s", originChainConfig.ChainName, destinationChainConfig.ChainName), zap.String("originDispatchTxHash", initiateTxHash), diff --git a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go index e3d749b..891dba5 100644 --- a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go +++ b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "github.com/skip-mev/go-fast-solver/shared/utils" "math/big" "strconv" "time" @@ -197,10 +198,23 @@ func (r *orderFulfillmentHandler) FillOrder( } txHash, rawTx, _, err := destinationChainBridgeClient.FillOrder(ctx, order, destinationChainGatewayContractAddress) - metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) if err != nil { return "", fmt.Errorf("filling order on destination chain at address %s: %w", destinationChainBridgeClient, err) } + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) + + destinationChainClient, err := r.clientManager.GetClient(ctx, order.DestinationChainID) + if err != nil { + lmt.Logger(ctx).Error("failed to get chain client to monitor gas balance", zap.Error(err)) + } + + // dont fail if we cant get the chain client, just log an error + if destinationChainClient != nil { + err = utils.MonitorGasBalance(ctx, order.DestinationChainID, destinationChainClient) + if err != nil { + lmt.Logger(ctx).Error("failed to monitor gas balance", zap.Error(err), zap.String("chainID", order.DestinationChainID)) + } + } if _, err := r.db.InsertSubmittedTx(ctx, db.InsertSubmittedTxParams{ OrderID: sql.NullInt64{Int64: order.ID, Valid: true}, @@ -397,9 +411,24 @@ func (r *orderFulfillmentHandler) InitiateTimeout(ctx context.Context, order db. } txHash, rawTx, _, err := destinationChainBridgeClient.InitiateTimeout(ctx, order, destinationChainGatewayContractAddress) + if err != nil { + return fmt.Errorf("error initiating timeout: %w", err) + } + metrics.FromContext(ctx).IncTransactionSubmitted(err == nil, order.SourceChainID, order.DestinationChainID) + + destinationChainClient, err := r.clientManager.GetClient(ctx, order.DestinationChainID) if err != nil { - return fmt.Errorf("initiating timeout: %w", err) + lmt.Logger(ctx).Error("failed to get destinationChainClient to monitor gas balance", + zap.Error(err), zap.String("chainID", order.DestinationChainID)) + } + + // dont fail if we cant get the source chain client, just log an error + if destinationChainClient != nil { + err = utils.MonitorGasBalance(ctx, order.DestinationChainID, destinationChainClient) + if err != nil { + lmt.Logger(ctx).Error("failed to monitor gas balance", zap.Error(err), zap.String("chainID", order.DestinationChainID)) + } } if _, err := r.db.InsertSubmittedTx(ctx, db.InsertSubmittedTxParams{ diff --git a/ordersettler/ordersettler.go b/ordersettler/ordersettler.go index 0e0bf51..2493fed 100644 --- a/ordersettler/ordersettler.go +++ b/ordersettler/ordersettler.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/skip-mev/go-fast-solver/shared/utils" "math/big" "strconv" "time" @@ -329,6 +330,20 @@ func (r *OrderSettler) SettleBatch(ctx context.Context, batch types.SettlementBa return fmt.Errorf("initiating batch settlement on chain %s: %w", batch.DestinationChainID(), err) } + sourceChainClient, err := r.clientManager.GetClient(ctx, batch.SourceChainID()) + if err != nil { + lmt.Logger(ctx).Error("failed to get chain client to monitor gas balance", + zap.Error(err), zap.String("chainID", batch.SourceChainID())) + } + + // dont fail if we cant get the chain client, just log an error + if sourceChainClient != nil { + err = utils.MonitorGasBalance(ctx, batch.SourceChainID(), sourceChainClient) + if err != nil { + lmt.Logger(ctx).Error("failed to monitor gas balance", zap.Error(err), zap.String("chainID", batch.SourceChainID())) + } + } + err = r.db.InTx(ctx, func(ctx context.Context, q db.Querier) error { for _, settlement := range batch { settlementTx := db.SetInitiateSettlementTxParams{ diff --git a/shared/bridges/cctp/bridge_client.go b/shared/bridges/cctp/bridge_client.go index d4e42c3..7a0a6b0 100644 --- a/shared/bridges/cctp/bridge_client.go +++ b/shared/bridges/cctp/bridge_client.go @@ -50,7 +50,7 @@ func (e ErrReceiveNotFound) Error() string { type BridgeClient interface { BlockHeight(ctx context.Context) (uint64, error) - SignerGasTokenBalance(ctx context.Context) (*big.Int, error) + GasTokenBalance(ctx context.Context) (*big.Int, error) FillOrder(ctx context.Context, order db.Order, gatewayContractAddress string) (string, string, *uint64, error) GetTxResult(ctx context.Context, txHash string) (*big.Int, *TxFailure, error) InitiateBatchSettlement(ctx context.Context, batch types.SettlementBatch) (string, string, error) diff --git a/shared/bridges/cctp/cosmos_bridge_client.go b/shared/bridges/cctp/cosmos_bridge_client.go index 9fe9695..f7ed6d8 100644 --- a/shared/bridges/cctp/cosmos_bridge_client.go +++ b/shared/bridges/cctp/cosmos_bridge_client.go @@ -131,8 +131,18 @@ func (c *CosmosBridgeClient) Balance( return response.Balance.Amount.BigInt(), nil } -func (c *CosmosBridgeClient) SignerGasTokenBalance(ctx context.Context) (*big.Int, error) { - return nil, errors.New("not implemented") +func (c *CosmosBridgeClient) GasTokenBalance(ctx context.Context) (*big.Int, error) { + fromAddress, err := bech32.ConvertAndEncode(c.prefix, c.signer.Address()) + if err != nil { + return nil, fmt.Errorf("converting signer address to bech32: %w", err) + } + + balance, err := c.Balance(ctx, fromAddress, c.gasDenom) + if err != nil { + return nil, fmt.Errorf("querying gas token balance: %w", err) + } + + return balance, nil } func (c *CosmosBridgeClient) Allowance(ctx context.Context, owner string) (*big.Int, error) { diff --git a/shared/bridges/cctp/evm_bridge_client.go b/shared/bridges/cctp/evm_bridge_client.go index f9f768b..20a2bf9 100644 --- a/shared/bridges/cctp/evm_bridge_client.go +++ b/shared/bridges/cctp/evm_bridge_client.go @@ -74,7 +74,7 @@ func (c *EVMBridgeClient) USDCBalance(ctx context.Context, address string) (*big return balance, nil } -func (c *EVMBridgeClient) SignerGasTokenBalance(ctx context.Context) (*big.Int, error) { +func (c *EVMBridgeClient) GasTokenBalance(ctx context.Context) (*big.Int, error) { balance, err := c.client.BalanceAt(ctx, c.fromAddress, nil) if err != nil { return nil, err diff --git a/shared/bridges/cctp/svm_bridge_client.go b/shared/bridges/cctp/svm_bridge_client.go index 558f411..2b9a4d8 100644 --- a/shared/bridges/cctp/svm_bridge_client.go +++ b/shared/bridges/cctp/svm_bridge_client.go @@ -66,10 +66,10 @@ func (c *SvmBridgeClient) IsSettlementComplete(ctx context.Context, gatewayContr } // Queries -func (c *SvmBridgeClient) SignerGasTokenBalance(ctx context.Context) (*big.Int, error) { +func (c *SvmBridgeClient) GasTokenBalance(ctx context.Context) (*big.Int, error) { balanceResult, err := c.rpcClient.GetBalance(ctx, c.signerAddress, rpc.CommitmentConfirmed) if err != nil { - return nil, fmt.Errorf("failed to get signer balance: %w", err) + return nil, fmt.Errorf("failed to get gas token balance: %w", err) } return big.NewInt(int64(balanceResult.Value)), nil diff --git a/shared/config/config.go b/shared/config/config.go index 20930b5..16960d2 100644 --- a/shared/config/config.go +++ b/shared/config/config.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "math/big" "os" "strings" "time" @@ -104,45 +105,49 @@ type RelayerConfig struct { MailboxAddress string `yaml:"mailbox_address"` } -type SignerGasBalanceConfig struct { +type GasBalanceConfig struct { WarningThresholdWei string `yaml:"warning_threshold_wei"` CriticalThresholdWei string `yaml:"critical_threshold_wei"` } type CosmosConfig struct { - RPC string `yaml:"rpc"` - RPCBasicAuthVar string `yaml:"rpc_basic_auth_var"` - GRPC string `yaml:"grpc"` - GRPCTLSEnabled bool `yaml:"grpc_tls_enabled"` - AddressPrefix string `yaml:"address_prefix"` - SignerGasBalance SignerGasBalanceConfig `yaml:"signer_gas_balance"` - USDCDenom string `yaml:"usdc_denom"` - GasPrice float64 `yaml:"gas_price"` - GasDenom string `yaml:"gas_denom"` + ChainID string `yaml:"chain_id"` + RPC string `yaml:"rpc"` + RPCBasicAuthVar string `yaml:"rpc_basic_auth_var"` + GRPC string `yaml:"grpc"` + GRPCTLSEnabled bool `yaml:"grpc_tls_enabled"` + AddressPrefix string `yaml:"address_prefix"` + GasBalance GasBalanceConfig `yaml:"gas_balance"` + USDCDenom string `yaml:"usdc_denom"` + SolverAddress string `yaml:"solver_address"` + GasPrice float64 `yaml:"gas_price"` + GasDenom string `yaml:"gas_denom"` } type EVMConfig struct { - MinGasTipCap *int64 `yaml:"min_gas_tip_cap"` - ChainID string `yaml:"chain_id"` - FastTransferContractAddress string `yaml:"fast_transfer_contract_address"` - RPC string `yaml:"rpc"` - RPCBasicAuthVar string `yaml:"rpc_basic_auth_var"` - GRPC string `yaml:"grpc"` - GRPCTLSEnabled bool `yaml:"grpc_tls_enabled"` - AddressPrefix string `yaml:"address_prefix"` - SignerGasBalance SignerGasBalanceConfig `yaml:"signer_gas_balance"` - SolverAddress string `yaml:"solver_address"` - USDCDenom string `yaml:"usdc_denom"` - Contracts ContractsConfig `yaml:"contracts"` + MinGasTipCap *int64 `yaml:"min_gas_tip_cap"` + ChainID string `yaml:"chain_id"` + FastTransferContractAddress string `yaml:"fast_transfer_contract_address"` + RPC string `yaml:"rpc"` + RPCBasicAuthVar string `yaml:"rpc_basic_auth_var"` + GRPC string `yaml:"grpc"` + GRPCTLSEnabled bool `yaml:"grpc_tls_enabled"` + AddressPrefix string `yaml:"address_prefix"` + GasBalance GasBalanceConfig `yaml:"gas_balance"` + SolverAddress string `yaml:"solver_address"` + USDCDenom string `yaml:"usdc_denom"` + Contracts ContractsConfig `yaml:"contracts"` } type SVMConfig struct { - RPC string `yaml:"rpc"` - WS string `yaml:"ws"` - SignerGasBalance SignerGasBalanceConfig `yaml:"signer_gas_balance"` - FastTransferContractAddress string `yaml:"fast_transfer_contract_address"` - PriorityFee uint64 `yaml:"priority_fee"` - SubmitRPCs []string `yaml:"submit_rpcs"` + ChainID string `yaml:"chain_id"` + RPC string `yaml:"rpc"` + WS string `yaml:"ws"` + GasBalance GasBalanceConfig `yaml:"gas_balance"` + SolverAddress string `yaml:"solver_address"` + FastTransferContractAddress string `yaml:"fast_transfer_contract_address"` + PriorityFee uint64 `yaml:"priority_fee"` + SubmitRPCs []string `yaml:"submit_rpcs"` } type ContractsConfig struct { @@ -195,13 +200,15 @@ type ConfigReader interface { GetChainConfig(chainID string) (ChainConfig, error) GetAllChainConfigsOfType(chainType ChainType) ([]ChainConfig, error) - + GetChainID(domain uint32, environment ChainEnvironment) (string, error) GetCoingeckoConfig() CoingeckoConfig GetGatewayContractAddress(chainID string) (string, error) GetChainIDByHyperlaneDomain(domain string) (string, error) GetUSDCDenom(chainID string) (string, error) + + GetGasAlertThresholds(chainID string) (warningThreshold, criticalThreshold *big.Int, err error) } type configReader struct { @@ -382,3 +389,59 @@ func (r configReader) GetUSDCDenom(chainID string) (string, error) { return "", fmt.Errorf("no usdc denom available for chain type %s", chainConfig.Type) } } + +func (r configReader) GetGasAlertThresholds(chainID string) (warningThreshold, criticalThreshold *big.Int, err error) { + var warningThresholdString, criticalThresholdString string + + chain, err := r.GetChainConfig(chainID) + if err != nil { + return nil, nil, err + } + switch chain.Type { + case ChainType_COSMOS: + warningThresholdString = chain.Cosmos.GasBalance.WarningThresholdWei + criticalThresholdString = chain.Cosmos.GasBalance.CriticalThresholdWei + case ChainType_EVM: + warningThresholdString = chain.EVM.GasBalance.WarningThresholdWei + criticalThresholdString = chain.EVM.GasBalance.CriticalThresholdWei + case ChainType_SVM: + warningThresholdString = "0" + criticalThresholdString = "0" + default: + return nil, nil, fmt.Errorf("unknown chain type") + } + + warningThreshold, ok := new(big.Int).SetString(warningThresholdString, 10) + if !ok { + return nil, nil, fmt.Errorf("failed to parse gas balance threshold amount") + } + criticalThreshold, ok = new(big.Int).SetString(criticalThresholdString, 10) + if !ok { + return nil, nil, fmt.Errorf("failed to parse gas balance threshold amount") + } + + return warningThreshold, criticalThreshold, nil +} + +func (r configReader) GetChainID(domain uint32, environment ChainEnvironment) (string, error) { + domainIndex, ok := r.cctpDomainIndex[environment] + if !ok { + return "", fmt.Errorf("cctp domain index not found for environment %s", environment) + } + + chain, ok := domainIndex[domain] + if !ok { + return "", fmt.Errorf("cctp domain %d not found for environment %s", domain, environment) + } + + switch chain.Type { + case ChainType_COSMOS: + return chain.Cosmos.ChainID, nil + case ChainType_EVM: + return chain.EVM.ChainID, nil + case ChainType_SVM: + return chain.SVM.ChainID, nil + default: + return "", fmt.Errorf("unknown chain type") + } +} diff --git a/shared/metrics/metrics.go b/shared/metrics/metrics.go index 9ac83a4..0f3f383 100644 --- a/shared/metrics/metrics.go +++ b/shared/metrics/metrics.go @@ -3,6 +3,8 @@ package metrics import ( "context" "fmt" + "math" + "math/big" "time" "github.com/go-kit/kit/metrics" @@ -12,6 +14,7 @@ import ( const ( chainIDLabel = "chain_id" + chainNameLabel = "chain_name" sourceChainIDLabel = "source_chain_id" destinationChainIDLabel = "destination_chain_id" successLabel = "success" @@ -19,6 +22,8 @@ const ( transferStatusLabel = "transfer_status" settlementStatusLabel = "settlement_status" operationLabel = "operation" + gasBalanceLevelLabel = "gas_balance_level" + gasTokenSymbolLabel = "gas_token_symbol" ) type Metrics interface { @@ -27,24 +32,26 @@ type Metrics interface { IncFillOrders(sourceChainID, destinationChainID, orderStatus string) DecFillOrders(sourceChainID, destinationChainID, orderStatus string) - ObserveFillLatency(sourceChainID, destinationChainID string, orderStatus string, latency time.Duration) + ObserveFillLatency(sourceChainID, destinationChainID, orderStatus string, latency time.Duration) IncOrderSettlements(sourceChainID, destinationChainID, settlementStatus string) DecOrderSettlements(sourceChainID, destinationChainID, settlementStatus string) - ObserveSettlementLatency(sourceChainID, destinationChainID string, settlementStatus string, latency time.Duration) + ObserveSettlementLatency(sourceChainID, destinationChainID, settlementStatus string, latency time.Duration) - IncFundsRebalanceTransfers(sourceChainID, destinationChainID string, transferStatus string) - DecFundsRebalanceTransfers(sourceChainID, destinationChainID string, transferStatus string) + IncFundsRebalanceTransfers(sourceChainID, destinationChainID, transferStatus string) + DecFundsRebalanceTransfers(sourceChainID, destinationChainID, transferStatus string) IncHyperlaneCheckpointingErrors() - IncHyperlaneMessages(sourceChainID, destinationChainID string, messageStatus string) - DecHyperlaneMessages(sourceChainID, destinationChainID string, messageStatus string) + IncHyperlaneMessages(sourceChainID, destinationChainID, messageStatus string) + DecHyperlaneMessages(sourceChainID, destinationChainID, messageStatus string) ObserveHyperlaneLatency(sourceChainID, destinationChainID, transferStatus string, latency time.Duration) ObserveTransferSizeExceeded(sourceChainID, destinationChainID string, amountExceededBy uint64) ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBpsExceededBy int64) IncDatabaseErrors(operation string) + + SetGasBalance(chainID, chainName, gasTokenSymbol string, gasBalance, warningThreshold, criticalThreshold big.Int, gasTokenDecimals uint8) } type metricsContextKey struct{} @@ -84,6 +91,7 @@ type PromMetrics struct { feeBpsRejections metrics.Histogram databaseErrors metrics.Counter + gasBalance metrics.Gauge } func NewPromMetrics() Metrics { @@ -166,6 +174,11 @@ func NewPromMetrics() Metrics { Name: "database_errors_total", Help: "number of errors encountered when making database calls", }, []string{}), + gasBalance: prom.NewGaugeFrom(stdprom.GaugeOpts{ + Namespace: "solver", + Name: "gas_balance_gauge", + Help: "gas balances, paginated by chain id and gas balance level", + }, []string{chainIDLabel, chainNameLabel, gasTokenSymbolLabel, gasBalanceLevelLabel}), } } @@ -241,6 +254,21 @@ func (m *PromMetrics) IncDatabaseErrors(operation string) { m.databaseErrors.With(operationLabel, operation).Add(1) } +func (m *PromMetrics) SetGasBalance(chainID, chainName, gasTokenSymbol string, gasBalance, warningThreshold, criticalThreshold big.Int, gasTokenDecimals uint8) { + gasBalanceLevel := "ok" + if gasBalance.Cmp(&warningThreshold) < 0 { + gasBalanceLevel = "warning" + } + if gasBalance.Cmp(&criticalThreshold) < 0 { + gasBalanceLevel = "critical" + } + // We compare the gas balance against thresholds locally rather than in the grafana alert definition since + // the prometheus metric is exported as a float64 and the thresholds reach Wei amounts where precision is lost. + gasBalanceFloat, _ := gasBalance.Float64() + gasTokenAmount := gasBalanceFloat / (math.Pow10(int(gasTokenDecimals))) + m.gasBalance.With(chainIDLabel, chainID, chainNameLabel, chainName, gasTokenSymbolLabel, gasTokenSymbol, gasBalanceLevelLabel, gasBalanceLevel).Set(gasTokenAmount) +} + type NoOpMetrics struct{} func (n NoOpMetrics) IncTransactionSubmitted(success bool, sourceChainID, destinationChainID string) { @@ -270,6 +298,9 @@ func (n NoOpMetrics) ObserveTransferSizeExceeded(sourceChainID, destinationChain } func (n NoOpMetrics) IncDatabaseErrors(operation string) {} func (n NoOpMetrics) ObserveFeeBpsRejection(sourceChainID, destinationChainID string, feeBps int64) {} +func (n *NoOpMetrics) SetGasBalance(chainID, chainName, gasTokenSymbol string, gasBalance, warningThreshold, criticalThreshold big.Int, gasTokenDecimals uint8) { +} + func NewNoOpMetrics() Metrics { return &NoOpMetrics{} } diff --git a/shared/utils/util.go b/shared/utils/util.go new file mode 100644 index 0000000..e53644a --- /dev/null +++ b/shared/utils/util.go @@ -0,0 +1,37 @@ +package utils + +import ( + "context" + "fmt" + "github.com/skip-mev/go-fast-solver/shared/bridges/cctp" + "github.com/skip-mev/go-fast-solver/shared/config" + "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" + "go.uber.org/zap" +) + +// MonitorGasBalance exports a metric indicating the current gas balance of the relayer signer and whether it is below alerting thresholds +func MonitorGasBalance(ctx context.Context, chainID string, chainClient cctp.BridgeClient) error { + balance, err := chainClient.GasTokenBalance(ctx) + if err != nil { + lmt.Logger(ctx).Error("failed to get gas token balance", zap.Error(err), zap.String("chain_id", chainID)) + return err + } + + chainConfig, err := config.GetConfigReader(ctx).GetChainConfig(chainID) + if err != nil { + return err + } + warningThreshold, criticalThreshold, err := config.GetConfigReader(ctx).GetGasAlertThresholds(chainID) + if err != nil { + return err + } + if balance == nil || warningThreshold == nil || criticalThreshold == nil { + return fmt.Errorf("gas balance or alert thresholds are nil for chain %s", chainID) + } + if balance.Cmp(criticalThreshold) < 0 { + lmt.Logger(ctx).Error("low balance", zap.String("balance", balance.String()), zap.String("chainID", chainID)) + } + metrics.FromContext(ctx).SetGasBalance(chainID, chainConfig.ChainName, chainConfig.GasTokenSymbol, *balance, *warningThreshold, *criticalThreshold, chainConfig.GasTokenDecimals) + return nil +} From f0d4063e18431ae5a535bbdc479040e0fb99e03e Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Thu, 31 Oct 2024 01:43:53 -0400 Subject: [PATCH 09/11] cleanup import order --- fundrebalancer/fundrebalancer.go | 4 ++-- hyperlane/relayer.go | 4 ++-- .../order_fulfillment_handler.go | 13 ++++++------- ordersettler/ordersettler.go | 17 +++++++---------- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/fundrebalancer/fundrebalancer.go b/fundrebalancer/fundrebalancer.go index 28ae3f3..f0edb1b 100644 --- a/fundrebalancer/fundrebalancer.go +++ b/fundrebalancer/fundrebalancer.go @@ -4,8 +4,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/skip-mev/go-fast-solver/shared/clientmanager" - "github.com/skip-mev/go-fast-solver/shared/utils" "math/big" "os" "time" @@ -14,12 +12,14 @@ import ( "github.com/skip-mev/go-fast-solver/shared/metrics" "github.com/skip-mev/go-fast-solver/db/gen/db" + "github.com/skip-mev/go-fast-solver/shared/clientmanager" "github.com/skip-mev/go-fast-solver/shared/clients/skipgo" "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/evmrpc" "github.com/skip-mev/go-fast-solver/shared/lmt" "github.com/skip-mev/go-fast-solver/shared/signing" "github.com/skip-mev/go-fast-solver/shared/signing/evm" + "github.com/skip-mev/go-fast-solver/shared/utils" "go.uber.org/zap" "golang.org/x/net/context" ) diff --git a/hyperlane/relayer.go b/hyperlane/relayer.go index 0a8ba5a..865c0bc 100644 --- a/hyperlane/relayer.go +++ b/hyperlane/relayer.go @@ -4,13 +4,13 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/skip-mev/go-fast-solver/shared/clientmanager" - "github.com/skip-mev/go-fast-solver/shared/utils" "strings" "github.com/ethereum/go-ethereum/crypto" "github.com/skip-mev/go-fast-solver/hyperlane/types" + "github.com/skip-mev/go-fast-solver/shared/clientmanager" + "github.com/skip-mev/go-fast-solver/shared/utils" "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/lmt" diff --git a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go index 891dba5..148a7d4 100644 --- a/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go +++ b/orderfulfiller/order_fulfillment_handler/order_fulfillment_handler.go @@ -4,28 +4,27 @@ import ( "context" "database/sql" "fmt" - "github.com/skip-mev/go-fast-solver/shared/utils" "math/big" "strconv" "time" dbtypes "github.com/skip-mev/go-fast-solver/db" + "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/orderfulfiller" "github.com/skip-mev/go-fast-solver/shared/bridges/cctp" "github.com/skip-mev/go-fast-solver/shared/clientmanager" - coingecko2 "github.com/skip-mev/go-fast-solver/shared/clients/coingecko" - "github.com/skip-mev/go-fast-solver/shared/metrics" - - "github.com/skip-mev/go-fast-solver/db/gen/db" + "github.com/skip-mev/go-fast-solver/shared/clients/coingecko" "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" + "github.com/skip-mev/go-fast-solver/shared/utils" "go.uber.org/zap" ) type orderFulfillmentHandler struct { db orderfulfiller.Database clientManager *clientmanager.ClientManager - PriceClient coingecko2.PriceClient + PriceClient coingecko.PriceClient } func NewOrderFulfillmentHandler(ctx context.Context, db orderfulfiller.Database, clientManager *clientmanager.ClientManager) (*orderFulfillmentHandler, error) { @@ -34,7 +33,7 @@ func NewOrderFulfillmentHandler(ctx context.Context, db orderfulfiller.Database, return &orderFulfillmentHandler{ db: db, clientManager: clientManager, - PriceClient: coingecko2.NewCachedPriceClient(coingecko2.DefaultCoingeckoClient(coingeckoConfig), coingeckoConfig.CacheRefreshInterval), + PriceClient: coingecko.NewCachedPriceClient(coingecko.DefaultCoingeckoClient(coingeckoConfig), coingeckoConfig.CacheRefreshInterval), }, nil } diff --git a/ordersettler/ordersettler.go b/ordersettler/ordersettler.go index 2493fed..d358146 100644 --- a/ordersettler/ordersettler.go +++ b/ordersettler/ordersettler.go @@ -5,24 +5,21 @@ import ( "database/sql" "errors" "fmt" - "github.com/skip-mev/go-fast-solver/shared/utils" "math/big" "strconv" "time" dbtypes "github.com/skip-mev/go-fast-solver/db" + "github.com/skip-mev/go-fast-solver/db/gen/db" "github.com/skip-mev/go-fast-solver/ordersettler/types" - "github.com/skip-mev/go-fast-solver/shared/metrics" - "golang.org/x/sync/errgroup" - "github.com/skip-mev/go-fast-solver/shared/clientmanager" - - coingecko2 "github.com/skip-mev/go-fast-solver/shared/clients/coingecko" - - "github.com/skip-mev/go-fast-solver/db/gen/db" + "github.com/skip-mev/go-fast-solver/shared/clients/coingecko" "github.com/skip-mev/go-fast-solver/shared/config" "github.com/skip-mev/go-fast-solver/shared/lmt" + "github.com/skip-mev/go-fast-solver/shared/metrics" + "github.com/skip-mev/go-fast-solver/shared/utils" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) type Config struct { @@ -52,7 +49,7 @@ type Database interface { type OrderSettler struct { db Database clientManager *clientmanager.ClientManager - PriceClient coingecko2.PriceClient + PriceClient coingecko.PriceClient } func NewOrderSettler(ctx context.Context, db Database, clientManager *clientmanager.ClientManager) (*OrderSettler, error) { @@ -61,7 +58,7 @@ func NewOrderSettler(ctx context.Context, db Database, clientManager *clientmana return &OrderSettler{ db: db, clientManager: clientManager, - PriceClient: coingecko2.NewCachedPriceClient(coingecko2.DefaultCoingeckoClient(coingeckoConfig), coingeckoConfig.CacheRefreshInterval), + PriceClient: coingecko.NewCachedPriceClient(coingecko.DefaultCoingeckoClient(coingeckoConfig), coingeckoConfig.CacheRefreshInterval), }, nil } From df3517737dc1aed83b803f288605b6e106717dab Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Thu, 31 Oct 2024 01:55:34 -0400 Subject: [PATCH 10/11] rename clientmanager to cctpClientManager --- fundrebalancer/fundrebalancer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fundrebalancer/fundrebalancer.go b/fundrebalancer/fundrebalancer.go index f0edb1b..ee34659 100644 --- a/fundrebalancer/fundrebalancer.go +++ b/fundrebalancer/fundrebalancer.go @@ -40,7 +40,7 @@ type FundRebalancer struct { chainIDToPrivateKey map[string]string skipgo skipgo.SkipGoClient evmClientManager evmrpc.EVMRPCClientManager - clientManager *clientmanager.ClientManager + cctpClientManager *clientmanager.ClientManager config map[string]config.FundRebalancerConfig database Database trasferTracker *TransferTracker @@ -51,7 +51,7 @@ func NewFundRebalancer( keysPath string, skipgo skipgo.SkipGoClient, evmClientManager evmrpc.EVMRPCClientManager, - clientmanager *clientmanager.ClientManager, + cctpClientManager *clientmanager.ClientManager, database Database, ) (*FundRebalancer, error) { chainIDToPriavateKey, err := loadChainIDToPrivateKeyMap(keysPath) @@ -63,7 +63,7 @@ func NewFundRebalancer( chainIDToPrivateKey: chainIDToPriavateKey, skipgo: skipgo, evmClientManager: evmClientManager, - clientManager: clientmanager, + cctpClientManager: cctpClientManager, config: config.GetConfigReader(ctx).Config().FundRebalancer, database: database, trasferTracker: NewTransferTracker(skipgo, database), @@ -262,7 +262,7 @@ func (r *FundRebalancer) MoveFundsToChain( return nil, nil, fmt.Errorf("submitting signed txns required for fund rebalancing: %w", err) } - sourceChainClient, err := r.clientManager.GetClient(ctx, rebalanceFromChainID) + sourceChainClient, err := r.cctpClientManager.GetClient(ctx, rebalanceFromChainID) if err != nil { lmt.Logger(ctx).Error("failed to get chain client to monitor gas balance", zap.Error(err)) } From e91fb211722073f6e7c081c6f9bcc131fc39487d Mon Sep 17 00:00:00 2001 From: nadimabdelaziz Date: Thu, 31 Oct 2024 02:09:45 -0400 Subject: [PATCH 11/11] fix tests --- fundrebalancer/fundrebalancer_test.go | 8 ++++---- shared/config/config.go | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fundrebalancer/fundrebalancer_test.go b/fundrebalancer/fundrebalancer_test.go index ac6c239..1684f30 100644 --- a/fundrebalancer/fundrebalancer_test.go +++ b/fundrebalancer/fundrebalancer_test.go @@ -136,7 +136,7 @@ func TestFundRebalancer_Rebalance(t *testing.T) { mockEVMClientManager.EXPECT().GetClient(mockContext, arbitrumChainID).Return(mockEVMClient, nil) mockDatabse := mock_database.NewMockDatabase(t) - rebalancer, err := fundrebalancer.NewFundRebalancer(ctx, f.Name(), mockSkipGo, mockEVMClientManager, mockDatabse) + rebalancer, err := fundrebalancer.NewFundRebalancer(ctx, f.Name(), mockSkipGo, mockEVMClientManager, nil, mockDatabse) assert.NoError(t, err) // setup initial state of mocks @@ -204,7 +204,7 @@ func TestFundRebalancer_Rebalance(t *testing.T) { mockEVMClientManager.EXPECT().GetClient(mockContext, arbitrumChainID).Return(mockEVMClient, nil) mockDatabse := mock_database.NewMockDatabase(t) - rebalancer, err := fundrebalancer.NewFundRebalancer(ctx, f.Name(), mockSkipGo, mockEVMClientManager, mockDatabse) + rebalancer, err := fundrebalancer.NewFundRebalancer(ctx, f.Name(), mockSkipGo, mockEVMClientManager, nil, mockDatabse) assert.NoError(t, err) // setup initial state of mocks @@ -316,7 +316,7 @@ func TestFundRebalancer_Rebalance(t *testing.T) { // using an in memory database for this test mockDatabse := mock_database.NewFakeDatabase() - rebalancer, err := fundrebalancer.NewFundRebalancer(ctx, f.Name(), mockSkipGo, mockEVMClientManager, mockDatabse) + rebalancer, err := fundrebalancer.NewFundRebalancer(ctx, f.Name(), mockSkipGo, mockEVMClientManager, nil, mockDatabse) assert.NoError(t, err) // setup initial state of mocks @@ -436,7 +436,7 @@ func TestFundRebalancer_Rebalance(t *testing.T) { mockEVMClientManager.EXPECT().GetClient(mockContext, arbitrumChainID).Return(mockEVMClient, nil) mockDatabse := mock_database.NewMockDatabase(t) - rebalancer, err := fundrebalancer.NewFundRebalancer(ctx, f.Name(), mockSkipGo, mockEVMClientManager, mockDatabse) + rebalancer, err := fundrebalancer.NewFundRebalancer(ctx, f.Name(), mockSkipGo, mockEVMClientManager, nil, mockDatabse) assert.NoError(t, err) // setup initial state of mocks diff --git a/shared/config/config.go b/shared/config/config.go index 16960d2..c685abc 100644 --- a/shared/config/config.go +++ b/shared/config/config.go @@ -190,7 +190,8 @@ func GetConfigReader(ctx context.Context) ConfigReader { } // Complex Config Queries - +// +//go:generate mockery --name=ConfigReader --filename=mock_config_reader.go --case=underscore type ConfigReader interface { Config() Config