feat(alerting): Implement alert-level provider overrides (#929)

* feat(alerting): Implement alert-level provider overrides

Fixes #96

* Fix tests

* Add missing test cases for alerting providers

* feat(alerting): Implement alert-level overrides on all providers

* chore: Add config.yaml to .gitignore

* fix typo in discord provider

* test: Start fixing tests for alerting providers

* test: Fix GitLab tests

* Fix all tests

* test: Improve coverage

* test: Improve coverage

* Rename override to provider-override

* docs: Mention new provider-override config

* test: Improve coverage

* test: Improve coverage

* chore: Rename Alert.OverrideAsBytes to Alert.ProviderOverrideAsBytes

Author: TwiN

.gitignore

@@ -17,4 +17,5 @@ node_modules
 *.db-shm
 *.db-wal
 gatus
-config/config.yml
+config/config.yml
+config.yaml

README.md

@@ -6,6 +6,9 @@ import (
 	"errors"
 	"strconv"
 	"strings"
+
+	"github.com/TwiN/logr"
+	"gopkg.in/yaml.v3"
 )
 
 var (
@@ -36,13 +39,17 @@ type Alert struct {
 	//
 	// This is a pointer, because it is populated by YAML and we need to know whether it was explicitly set to a value
 	// or not for provider.ParseWithDefaultAlert to work.
-	Description *string `yaml:"description"`
+	Description *string `yaml:"description,omitempty"`
 
 	// SendOnResolved defines whether to send a second notification when the issue has been resolved
 	//
 	// This is a pointer, because it is populated by YAML and we need to know whether it was explicitly set to a value
 	// or not for provider.ParseWithDefaultAlert to work. Use Alert.IsSendingOnResolved() for a non-pointer
-	SendOnResolved *bool `yaml:"send-on-resolved"`
+	SendOnResolved *bool `yaml:"send-on-resolved,omitempty"`
+
+	// ProviderOverride is an optional field that can be used to override the provider's configuration
+	// It is freeform so that it can be used for any provider-specific configuration.
+	ProviderOverride map[string]any `yaml:"provider-override,omitempty"`
 
 	// ResolveKey is an optional field that is used by some providers (i.e. PagerDuty's dedup_key) to resolve
 	// ongoing/triggered incidents
@@ -111,3 +118,11 @@ func (alert *Alert) Checksum() string {
 	)
 	return hex.EncodeToString(hash.Sum(nil))
 }
+
+func (alert *Alert) ProviderOverrideAsBytes() []byte {
+	yamlBytes, err := yaml.Marshal(alert.ProviderOverride)
+	if err != nil {
+		logr.Warnf("[alert.ProviderOverrideAsBytes] Failed to marshal alert override of type=%s as bytes: %v", alert.Type, err)
+	}
+	return yamlBytes
+}

alerting/alert/alert.go

@@ -1,30 +1,73 @@
 package awsses
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/TwiN/gatus/v5/alerting/alert"
 	"github.com/TwiN/gatus/v5/config/endpoint"
+	"github.com/TwiN/logr"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ses"
+	"gopkg.in/yaml.v3"
 )
 
 const (
 	CharSet = "UTF-8"
 )
 
-// AlertProvider is the configuration necessary for sending an alert using AWS Simple Email Service
-type AlertProvider struct {
+var (
+	ErrDuplicateGroupOverride = errors.New("duplicate group override")
+	ErrMissingFromOrToFields  = errors.New("from and to fields are required")
+	ErrInvalidAWSAuthConfig   = errors.New("either both or neither of access-key-id and secret-access-key must be specified")
+)
+
+type Config struct {
 	AccessKeyID     string `yaml:"access-key-id"`
 	SecretAccessKey string `yaml:"secret-access-key"`
 	Region          string `yaml:"region"`
 
 	From string `yaml:"from"`
 	To   string `yaml:"to"`
+}
+
+func (cfg *Config) Validate() error {
+	if len(cfg.From) == 0 || len(cfg.To) == 0 {
+		return ErrMissingFromOrToFields
+	}
+	if !((len(cfg.AccessKeyID) == 0 && len(cfg.SecretAccessKey) == 0) || (len(cfg.AccessKeyID) > 0 && len(cfg.SecretAccessKey) > 0)) {
+		// if both AccessKeyID and SecretAccessKey are specified, we'll use these to authenticate,
+		// otherwise if neither are specified, then we'll fall back on IAM authentication.
+		return ErrInvalidAWSAuthConfig
+	}
+	return nil
+}
+
+func (cfg *Config) Merge(override *Config) {
+	if len(override.AccessKeyID) > 0 {
+		cfg.AccessKeyID = override.AccessKeyID
+	}
+	if len(override.SecretAccessKey) > 0 {
+		cfg.SecretAccessKey = override.SecretAccessKey
+	}
+	if len(override.Region) > 0 {
+		cfg.Region = override.Region
+	}
+	if len(override.From) > 0 {
+		cfg.From = override.From
+	}
+	if len(override.To) > 0 {
+		cfg.To = override.To
+	}
+}
+
+// AlertProvider is the configuration necessary for sending an alert using AWS Simple Email Service
+type AlertProvider struct {
+	DefaultConfig Config `yaml:",inline"`
 
 	// DefaultAlert is the default alert configuration to use for endpoints with an alert of the appropriate type
 	DefaultAlert *alert.Alert `yaml:"default-alert,omitempty"`
@@ -35,36 +78,37 @@ type AlertProvider struct {
 
 // Override is a case under which the default integration is overridden
 type Override struct {
-	Group string `yaml:"group"`
-	To    string `yaml:"to"`
+	Group  string `yaml:"group"`
+	Config `yaml:",inline"`
 }
 
-// IsValid returns whether the provider's configuration is valid
-func (provider *AlertProvider) IsValid() bool {
+// Validate the provider's configuration
+func (provider *AlertProvider) Validate() error {
 	registeredGroups := make(map[string]bool)
 	if provider.Overrides != nil {
 		for _, override := range provider.Overrides {
 			if isAlreadyRegistered := registeredGroups[override.Group]; isAlreadyRegistered || override.Group == "" || len(override.To) == 0 {
-				return false
+				return ErrDuplicateGroupOverride
 			}
 			registeredGroups[override.Group] = true
 		}
 	}
-	// if both AccessKeyID and SecretAccessKey are specified, we'll use these to authenticate,
-	// otherwise if neither are specified, then we'll fall back on IAM authentication.
-	return len(provider.From) > 0 && len(provider.To) > 0 &&
-		((len(provider.AccessKeyID) == 0 && len(provider.SecretAccessKey) == 0) || (len(provider.AccessKeyID) > 0 && len(provider.SecretAccessKey) > 0))
+	return provider.DefaultConfig.Validate()
 }
 
 // Send an alert using the provider
 func (provider *AlertProvider) Send(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) error {
-	sess, err := provider.createSession()
+	cfg, err := provider.GetConfig(ep.Group, alert)
 	if err != nil {
 		return err
 	}
-	svc := ses.New(sess)
+	awsSession, err := provider.createSession(cfg)
+	if err != nil {
+		return err
+	}
+	svc := ses.New(awsSession)
 	subject, body := provider.buildMessageSubjectAndBody(ep, alert, result, resolved)
-	emails := strings.Split(provider.getToForGroup(ep.Group), ",")
+	emails := strings.Split(cfg.To, ",")
 
 	input := &ses.SendEmailInput{
 		Destination: &ses.Destination{
@@ -82,33 +126,41 @@ func (provider *AlertProvider) Send(ep *endpoint.Endpoint, alert *alert.Alert, r
 				Data:    aws.String(subject),
 			},
 		},
-		Source: aws.String(provider.From),
+		Source: aws.String(cfg.From),
 	}
-	_, err = svc.SendEmail(input)
-
-	if err != nil {
+	if _, err = svc.SendEmail(input); err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
 			switch aerr.Code() {
 			case ses.ErrCodeMessageRejected:
-				fmt.Println(ses.ErrCodeMessageRejected, aerr.Error())
+				logr.Error(ses.ErrCodeMessageRejected + ": " + aerr.Error())
 			case ses.ErrCodeMailFromDomainNotVerifiedException:
-				fmt.Println(ses.ErrCodeMailFromDomainNotVerifiedException, aerr.Error())
+				logr.Error(ses.ErrCodeMailFromDomainNotVerifiedException + ": " + aerr.Error())
 			case ses.ErrCodeConfigurationSetDoesNotExistException:
-				fmt.Println(ses.ErrCodeConfigurationSetDoesNotExistException, aerr.Error())
+				logr.Error(ses.ErrCodeConfigurationSetDoesNotExistException + ": " + aerr.Error())
 			default:
-				fmt.Println(aerr.Error())
+				logr.Error(aerr.Error())
 			}
 		} else {
 			// Print the error, cast err to awserr.Error to get the Code and
 			// Message from an error.
-			fmt.Println(err.Error())
+			logr.Error(err.Error())
 		}
 
 		return err
 	}
 	return nil
 }
 
+func (provider *AlertProvider) createSession(cfg *Config) (*session.Session, error) {
+	awsConfig := &aws.Config{
+		Region: aws.String(cfg.Region),
+	}
+	if len(cfg.AccessKeyID) > 0 && len(cfg.SecretAccessKey) > 0 {
+		awsConfig.Credentials = credentials.NewStaticCredentials(cfg.AccessKeyID, cfg.SecretAccessKey, "")
+	}
+	return session.NewSession(awsConfig)
+}
+
 // buildMessageSubjectAndBody builds the message subject and body
 func (provider *AlertProvider) buildMessageSubjectAndBody(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) (string, string) {
 	var subject, message string
@@ -139,29 +191,38 @@ func (provider *AlertProvider) buildMessageSubjectAndBody(ep *endpoint.Endpoint,
 	return subject, message + description + formattedConditionResults
 }
 
-// getToForGroup returns the appropriate email integration to for a given group
-func (provider *AlertProvider) getToForGroup(group string) string {
+// GetDefaultAlert returns the provider's default alert configuration
+func (provider *AlertProvider) GetDefaultAlert() *alert.Alert {
+	return provider.DefaultAlert
+}
+
+// GetConfig returns the configuration for the provider with the overrides applied
+func (provider *AlertProvider) GetConfig(group string, alert *alert.Alert) (*Config, error) {
+	cfg := provider.DefaultConfig
+	// Handle group overrides
 	if provider.Overrides != nil {
 		for _, override := range provider.Overrides {
 			if group == override.Group {
-				return override.To
+				cfg.Merge(&override.Config)
+				break
 			}
 		}
 	}
-	return provider.To
-}
-
-// GetDefaultAlert returns the provider's default alert configuration
-func (provider *AlertProvider) GetDefaultAlert() *alert.Alert {
-	return provider.DefaultAlert
+	// Handle alert overrides
+	if len(alert.ProviderOverride) != 0 {
+		overrideConfig := Config{}
+		if err := yaml.Unmarshal(alert.ProviderOverrideAsBytes(), &overrideConfig); err != nil {
+			return nil, err
+		}
+		cfg.Merge(&overrideConfig)
+	}
+	// Validate the configuration
+	err := cfg.Validate()
+	return &cfg, err
 }
 
-func (provider *AlertProvider) createSession() (*session.Session, error) {
-	config := &aws.Config{
-		Region: aws.String(provider.Region),
-	}
-	if len(provider.AccessKeyID) > 0 && len(provider.SecretAccessKey) > 0 {
-		config.Credentials = credentials.NewStaticCredentials(provider.AccessKeyID, provider.SecretAccessKey, "")
-	}
-	return session.NewSession(config)
+// ValidateOverrides validates the alert's provider override and, if present, the group override
+func (provider *AlertProvider) ValidateOverrides(group string, alert *alert.Alert) error {
+	_, err := provider.GetConfig(group, alert)
+	return err
 }