Resend failed Slack notifications

Author: CHTJonas

go.mod

@@ -1,8 +1,11 @@
 module github.com/CHTJonas/uptime-mon
 
-go 1.17
+go 1.23
+
+toolchain go1.24.0
 
 require (
+	github.com/cenkalti/backoff/v5 v5.0.2
 	github.com/slack-go/slack v0.12.1
 	github.com/spf13/viper v1.9.0
 )

go.sum

@@ -51,6 +51,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
+github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=

main.go

@@ -53,28 +53,19 @@ func testLoop() {
 				if strings.Contains(err.Error(), "Client.Timeout") {
 					err = fmt.Errorf("response time was greater than %d milliseconds", t.MaxResponseTime)
 				}
-				errStr := fmt.Sprintf("Test failed: %s: %s", t.Name, err)
-				debugPrintLn(errStr)
+				debugPrintf("Test failed: %s: %s\n", t.Name, err)
 				if t.ShouldNotify() && !t.notified {
-					err := notify(errStr)
-					if err != nil {
-						fmt.Println("error sending test failure Slack notification:", err)
-					} else {
-						debugPrintLn("successfully sent test failure Slack notification")
-						t.notified = true
-					}
+					msg := getMessage("failed", t.Name, err)
+					go notifyRetry(msg)
+					t.notified = true
 				}
 			} else {
 				debugPrintLn("Test successful:", t.Name)
 				if t.notified {
-					err := notifyf("Test recovered: %s", t.Name)
-					if err != nil {
-						fmt.Println("error sending test recovery Slack notification:", err)
-					} else {
-						debugPrintLn("successfully sent test recovery Slack notification")
-					}
+					msg := getMessage("recovered", t.Name, nil)
+					go notifyRetry(msg)
+					t.notified = false
 				}
-				t.notified = false
 			}
 		}(c.tests[i])
 		time.Sleep(duration)

slack.go

@@ -1,18 +1,45 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	"time"
 
+	"github.com/cenkalti/backoff/v5"
 	"github.com/slack-go/slack"
 )
 
+func getMessage(verb, name string, err error) string {
+	timeStr := time.Now().UTC().Format("2006-01-02 15:04:05")
+	if err == nil {
+		return fmt.Sprintf("Test %s @ %s\n%s", verb, timeStr, name)
+	}
+	return fmt.Sprintf("Test %s @ %s\n%s: %s", verb, timeStr, name, err)
+}
+
+func notifyRetry(msg string) error {
+	operation := func() (string, error) {
+		err := notify(msg)
+		if err != nil {
+			fmt.Println("error sending Slack notification:", err)
+			return "", err
+		}
+		debugPrintLn("successfully sent Slack notification")
+		return "", nil
+	}
+	strat := backoff.WithBackOff(&backoff.ExponentialBackOff{
+		InitialInterval:     3 * time.Second,
+		RandomizationFactor: 0.5,
+		Multiplier:          1.5,
+		MaxInterval:         48 * time.Hour,
+	})
+	_, err := backoff.Retry(context.TODO(), operation, strat)
+	return err
+}
+
 func notify(msg string) error {
 	hookURL := getConfig().slackWebhook
 	return slack.PostWebhook(hookURL, &slack.WebhookMessage{
 		Text: msg,
 	})
 }
-
-func notifyf(format string, a ...interface{}) error {
-	return notify(fmt.Sprintf(format, a...))
-}

utils.go

@@ -17,3 +17,10 @@ func debugPrintLn(a ...any) (n int, err error) {
 	}
 	return 0, nil
 }
+
+func debugPrintf(format string, a ...any) (n int, err error) {
+	if version == "dev" {
+		fmt.Printf(format, a...)
+	}
+	return 0, nil
+}