fix: clean up spam error logs

hyperlane/relayer.go

@@ -33,7 +33,8 @@ func NewRelayer(hyperlaneClient Client, storageLocationOverrides map[string]stri
 }
 
 var (
-	ErrMessageAlreadyDelivered = fmt.Errorf("message has already been delivered")
+	ErrMessageAlreadyDelivered  = fmt.Errorf("message has already been delivered")
+	ErrNotEnoughSignaturesFound = errors.New("signatures found below expected threshold in in multisig signed checkpoint")
 )
 
 func (r *relayer) Relay(ctx context.Context, originChainID string, initiateTxHash string) (destinationTxHash string, destinationChainID string, err error) {
@@ -215,7 +216,10 @@ func (r *relayer) checkpointAtIndex(
 		}
 	}
 	if len(multiSigCheckpoint.Signatures) < int(threshold) {
-		return types.MultiSigSignedCheckpoint{}, fmt.Errorf("expected atleast %d signatures in multisig signed checkpoint, but got %d", threshold, len(multiSigCheckpoint.Signatures))
+		lmt.Logger(ctx).Warn("failed to find expected number of signatures in multisig signed checkpoint",
+			zap.Uint8("threshold", threshold), zap.Int("num_signatures_found", len(multiSigCheckpoint.Signatures)))
+
+		return types.MultiSigSignedCheckpoint{}, ErrNotEnoughSignaturesFound
 	}
 	if strings.TrimPrefix(multiSigCheckpoint.Checkpoint.MessageID, "0x") != strings.TrimPrefix(messageID, "0x") {
 		return types.MultiSigSignedCheckpoint{}, fmt.Errorf("mismatch message id in checkpoint and dipsatch message. dispatch has %s and checkpoint has %s", messageID, multiSigCheckpoint.Checkpoint.MessageID)

hyperlane/relayer_runner.go

@@ -83,8 +83,46 @@ func (r *RelayerRunner) Run(ctx context.Context) error {
 					continue
 				}
 
+				isTransferValid, err := r.validateHyperlaneTransfer(ctx, transfer)
+				if err != nil {
+					lmt.Logger(ctx).Warn(
+						"failed to validate Hyperlane transfer",
+						zap.Error(err),
+						zap.Int64("transferId", transfer.ID),
+						zap.String("txHash", transfer.MessageSentTx),
+					)
+					continue
+				}
+
+				if !isTransferValid {
+					lmt.Logger(ctx).Warn(
+						"abandoning invalid hyperlane transfer",
+						zap.Any("transfer", transfer),
+						zap.Error(err),
+					)
+
+					if _, err := r.db.SetMessageStatus(ctx, db.SetMessageStatusParams{
+						TransferStatus:     dbtypes.TransferStatusAbandoned,
+						SourceChainID:      transfer.SourceChainID,
+						DestinationChainID: transfer.DestinationChainID,
+						MessageID:          transfer.MessageID,
+					}); err != nil {
+						lmt.Logger(ctx).Error(
+							"error updating invalid transfer status",
+							zap.Error(err),
+							zap.Int64("transferId", transfer.ID),
+							zap.String("txHash", transfer.MessageSentTx),
+						)
+					}
+					continue
+				}
+
 				destinationTxHash, destinationChainID, err := r.relayHandler.Relay(ctx, transfer.SourceChainID, transfer.MessageSentTx)
 				if err != nil {
+					if errors.Is(err, ErrNotEnoughSignaturesFound) {
+						// warning already logged in relayer
+						continue
+					}
 					lmt.Logger(ctx).Error(
 						"error relaying pending hyperlane transfer",
 						zap.Error(err),
@@ -250,3 +288,37 @@ func (r *RelayerRunner) findTimeoutsToRelay(ctx context.Context) error {
 	}
 	return nil
 }
+
+func (r *RelayerRunner) validateHyperlaneTransfer(ctx context.Context, transfer db.HyperlaneTransfer) (bool, error) {
+	destinationChainConfig, err := config.GetConfigReader(ctx).GetChainConfig(transfer.DestinationChainID)
+	if err != nil {
+		return false, fmt.Errorf("getting destination chain config: %w", err)
+	}
+
+	dispatch, _, err := r.hyperlane.GetHyperlaneDispatch(ctx, destinationChainConfig.HyperlaneDomain, transfer.DestinationChainID, transfer.MessageSentTx)
+	if err != nil {
+		return false, fmt.Errorf("getting hyperlane dispatch: %w", err)
+	}
+
+	if len(dispatch.Recipient) == 0 {
+		return false, fmt.Errorf("invalid hyperlane transfer: empty recipient")
+	}
+
+	if len(dispatch.Message) == 0 {
+		return false, fmt.Errorf("invalid hyperlane transfer: empty message")
+	}
+
+	if len(dispatch.Sender) == 0 {
+		return false, fmt.Errorf("invalid hyperlane transfer: empty sender")
+	}
+
+	if len(dispatch.MessageID) == 0 {
+		return false, fmt.Errorf("invalid hyperlane transfer: empty messageID")
+	}
+
+	if len(dispatch.DestinationDomain) == 0 {
+		return false, fmt.Errorf("invalid hyperlane transfer: empty destination domain")
+	}
+
+	return true, nil
+}

ordersettler/ordersettler.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"math/big"
 	"strconv"
+	"strings"
 	"time"
 
 	dbtypes "github.com/skip-mev/go-fast-solver/db"
@@ -215,7 +216,7 @@ func (r *OrderSettler) verifyOrderSettlements(ctx context.Context) error {
 
 		if err = r.verifyOrderSettlement(ctx, settlement); err != nil {
 			lmt.Logger(ctx).Warn(
-				"error verifying order settlement",
+				"failed to verify order settlement, will retry verification on next interval",
 				zap.Error(err),
 				zap.String("orderID", settlement.OrderID),
 				zap.String("sourceChainID", settlement.SourceChainID),
@@ -395,6 +396,11 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db.
 	if settlement.SettlementStatus == dbtypes.SettlementStatusPending {
 		gasCost, failure, err := destinationBridgeClient.GetTxResult(ctx, settlement.InitiateSettlementTx.String)
 		if err != nil {
+			// Check if the error is due to tx not found
+			if strings.Contains(err.Error(), "tx") && strings.Contains(err.Error(), "not found") && strings.Contains(err.Error(), settlement.InitiateSettlementTx.String) {
+				// Transaction not yet indexed, we'll check again later
+				return fmt.Errorf("transaction not yet indexed. will retry fetching next interval")
+			}
 			return fmt.Errorf("failed to fetch message received event: %w", err)
 		} else if failure != nil {
 			lmt.Logger(ctx).Error("tx failed", zap.String("failure", failure.String()))