Skip to content

Commit 10cabb9

Browse files
committed
fix(alerting): Prevent >2nd reminders from being skipped if the first one triggered
See #1226 (comment)
1 parent 3580bbb commit 10cabb9

File tree

1 file changed

+6
-1
lines changed

1 file changed

+6
-1
lines changed

watchdog/alerting.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ func HandleAlerting(ep *endpoint.Endpoint, result *endpoint.Result, alertingConf
2727
func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alertingConfig *alerting.Config) {
2828
ep.NumberOfSuccessesInARow = 0
2929
ep.NumberOfFailuresInARow++
30+
// Store the current LastReminderSent time so all alert providers use the same reference time for reminder checks
31+
// This is important in case there are multiple alerts: if the first one sends a reminder, it would update the value
32+
// of ep.LastReminderSent (since ep is a pointer), so the second one would never send a reminder, even if it was due.
33+
// By storing the value in a local variable, we ensure all alerts use the same reference
34+
lastReminderSent := ep.LastReminderSent
3035
for _, endpointAlert := range ep.Alerts {
3136
// If the alert hasn't been triggered, move to the next one
3237
if !endpointAlert.IsEnabled() || endpointAlert.FailureThreshold > ep.NumberOfFailuresInARow {
@@ -35,7 +40,7 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert
3540
// Determine if an initial alert should be sent
3641
sendInitialAlert := !endpointAlert.Triggered
3742
// Determine if a reminder should be sent
38-
sendReminder := endpointAlert.Triggered && endpointAlert.MinimumReminderInterval > 0 && time.Since(ep.LastReminderSent) >= endpointAlert.MinimumReminderInterval
43+
sendReminder := endpointAlert.Triggered && endpointAlert.MinimumReminderInterval > 0 && time.Since(lastReminderSent) >= endpointAlert.MinimumReminderInterval
3944
// If neither initial alert nor reminder needs to be sent, skip to the next alert
4045
if !sendInitialAlert && !sendReminder {
4146
logr.Debugf("[watchdog.handleAlertsToTrigger] Alert for endpoint=%s with description='%s' is not due for triggering or reminding, skipping", ep.Name, endpointAlert.GetDescription())

0 commit comments

Comments
 (0)