Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: clean up spam error logs #23

Merged
merged 9 commits into from
Nov 8, 2024
Merged

fix: clean up spam error logs #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
6f6669d
fix: validate hpl transfer before attempting relay
nadim-az Nov 8, 2024
ac39e95
update log message
nadim-az Nov 8, 2024
990e639
more log messages updated
nadim-az Nov 8, 2024
4c418a6
dont log error messages if cant find expected num of signatures
nadim-az Nov 8, 2024
4b0e782
add back error message
nadim-az Nov 8, 2024
5c2523d
more fixes
nadim-az Nov 8, 2024
d4cd1e7
update log messages
nadim-az Nov 8, 2024
907407f
pr feedback
nadim-az Nov 8, 2024
408685c
Merge branch 'main' into na/cleanup-logs
nadim-az Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions hyperlane/relayer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ func NewRelayer(hyperlaneClient Client, storageLocationOverrides map[string]stri
}

var (
ErrMessageAlreadyDelivered = fmt.Errorf("message has already been delivered")
ErrMessageAlreadyDelivered = fmt.Errorf("message has already been delivered")
ErrNotEnoughSignaturesFound = errors.New("number of signatures found in multisig signed checkpoint is below expected threshold")
)

func (r *relayer) Relay(ctx context.Context, originChainID string, initiateTxHash string) (destinationTxHash string, destinationChainID string, err error) {
Expand Down Expand Up @@ -162,7 +163,7 @@ func (r *relayer) checkpointAtIndex(
signedCheckpoint, err := fetcher.Checkpoint(ctx, index)
if errors.Is(err, ErrCheckpointDoesNotExist) {
// if the validator for this fetcher has not signed the
// chekcpoint, ignore it
// checkpoint, ignore it
continue
}
if err != nil {
Expand Down Expand Up @@ -215,7 +216,10 @@ func (r *relayer) checkpointAtIndex(
}
}
if len(multiSigCheckpoint.Signatures) < int(threshold) {
return types.MultiSigSignedCheckpoint{}, fmt.Errorf("expected atleast %d signatures in multisig signed checkpoint, but got %d", threshold, len(multiSigCheckpoint.Signatures))
lmt.Logger(ctx).Warn("failed to find expected number of signatures in multisig signed checkpoint",
zap.Uint8("threshold", threshold), zap.Int("num_signatures_found", len(multiSigCheckpoint.Signatures)))

return types.MultiSigSignedCheckpoint{}, ErrNotEnoughSignaturesFound
}
if strings.TrimPrefix(multiSigCheckpoint.Checkpoint.MessageID, "0x") != strings.TrimPrefix(messageID, "0x") {
return types.MultiSigSignedCheckpoint{}, fmt.Errorf("mismatch message id in checkpoint and dipsatch message. dispatch has %s and checkpoint has %s", messageID, multiSigCheckpoint.Checkpoint.MessageID)
Expand Down
31 changes: 31 additions & 0 deletions hyperlane/relayer_runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
"strings"
"time"

dbtypes "github.com/skip-mev/go-fast-solver/db"
Expand Down Expand Up @@ -85,6 +86,36 @@ func (r *RelayerRunner) Run(ctx context.Context) error {

destinationTxHash, destinationChainID, err := r.relayHandler.Relay(ctx, transfer.SourceChainID, transfer.MessageSentTx)
if err != nil {
// Unrecoverable error
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

{"level":"error","ts":1731035755.109063,"caller":"hyperlane/relayer_runner.go:88","msg":"error relaying pending hyperlane transfer","error":"processing message on domain 42161: processing message on destination mailbox: execution reverted: !threshold","sourceChainID":"osmosis-1","txHash":"9C4EDA7213A3909E7F852B4047BFE74F76CD2477C8C92CA1975DD70B74547F73","stacktrace":"[github.com/skip-mev/go-fast-solver/hyperlane.(*RelayerRunner).Run](http://github.com/skip-mev/go-fast-solver/hyperlane.(*RelayerRunner).Run)\n\t/solver/hyperlane/relayer_runner.go:88\nmain.main.func6\n\t/solver/cmd/solver/main.go:153\[ngolang.org/x/sync/errgroup.(*Group).Go.func1](http://ngolang.org/x/sync/errgroup.(*Group).Go.func1)\n\t/root/.cache/go-build/golang.org/x/[email protected]/errgroup/errgroup.go:78"}

if strings.Contains("execution reverted", err.Error()) {
lmt.Logger(ctx).Warn(
"abandoning hyperlane transfer",
zap.Int64("transferId", transfer.ID),
zap.String("txHash", transfer.MessageSentTx),
zap.Error(err),
)

if _, err := r.db.SetMessageStatus(ctx, db.SetMessageStatusParams{
TransferStatus: dbtypes.TransferStatusAbandoned,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you also fill the TransferStatusMessage field here with the revert message? bonus if you also rename this query method to SetTransferStatus

SourceChainID: transfer.SourceChainID,
DestinationChainID: transfer.DestinationChainID,
MessageID: transfer.MessageID,
}); err != nil {
lmt.Logger(ctx).Error(
"error updating invalid transfer status",
zap.Int64("transferId", transfer.ID),
zap.String("txHash", transfer.MessageSentTx),
zap.Error(err),
)
}
continue
}

// warning already logged in relayer
if errors.Is(err, ErrNotEnoughSignaturesFound) {
continue
}

lmt.Logger(ctx).Error(
"error relaying pending hyperlane transfer",
zap.Error(err),
Expand Down
8 changes: 7 additions & 1 deletion ordersettler/ordersettler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"fmt"
"math/big"
"strconv"
"strings"
"time"

dbtypes "github.com/skip-mev/go-fast-solver/db"
Expand Down Expand Up @@ -226,7 +227,7 @@ func (r *OrderSettler) verifyOrderSettlements(ctx context.Context) error {

if err = r.verifyOrderSettlement(ctx, settlement); err != nil {
lmt.Logger(ctx).Warn(
"error verifying order settlement",
"failed to verify order settlement, will retry verification on next interval",
zap.Error(err),
zap.String("orderID", settlement.OrderID),
zap.String("sourceChainID", settlement.SourceChainID),
Expand Down Expand Up @@ -413,6 +414,11 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db.
if settlement.SettlementStatus == dbtypes.SettlementStatusPending {
gasCost, failure, err := destinationBridgeClient.GetTxResult(ctx, settlement.InitiateSettlementTx.String)
if err != nil {
// Check if the error is due to tx not found
if strings.Contains(err.Error(), "tx") && strings.Contains(err.Error(), "not found") && strings.Contains(err.Error(), settlement.InitiateSettlementTx.String) {
// Transaction not yet indexed, we'll check again later
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

{"level":"warn","ts":1730925413.9067168,"caller":"ordersettler/ordersettler.go:218","msg":"error verifying order settlement","error":"failed to fetch message received event: error in json rpc client, with http response metadata: (Status: 200 OK, Protocol HTTP/2.0). RPC error -32603 - Internal error: tx (47F32765BE617BAC646C14556D639EB23785FEA6F7A9E2F25C0ADA41C92263F9) not found","orderID":"e39a4116fb7dddae48bcd40b69edaddc190b2ad098a72260db23f96805486c6b","sourceChainID":"42161"}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

damn so the node is returning a 200 but its actually an error. error string checking is never great but seems like this is probably the easiest solution for now so im good with it

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah i couldnt think of a cleaner way to do this

return fmt.Errorf("transaction not yet indexed. will retry fetching next interval")
}
return fmt.Errorf("failed to fetch message received event: %w", err)
} else if failure != nil {
lmt.Logger(ctx).Error("tx failed", zap.String("failure", failure.String()))
Expand Down
Loading