mon-alert-cfg.yml

apiVersion: v1
kind: ConfigMap
metadata:
  name: alertmanager
  namespace: <namespace>
  labels:
    app: monitor
    name: alertmanager
data:
  config.yml: |-
    global:

    ##
    # tool to help visualize where your alerts are going
    # https://prometheus.io/webtools/alerting/routing-tree-editor/
    ##

    # The directory from which notification templates are read.
    templates:
    - '/etc/alertmanager/template/*.tmpl'
    
    # The root route on which each incoming alert enters.
    route:
      # The labels by which incoming alerts are grouped together. For example,
      # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
      # be batched into a single group.
      group_by: ['service']
    
      # When a new group of alerts is created by an incoming alert, wait at
      # least 'group_wait' to send the initial notification.
      # This way ensures that you get multiple alerts for the same group that start
      # firing shortly after another are batched together on the first
      # notification.
      group_wait: 1m
    
      # When the first notification was sent, wait 'group_interval' to send a batch
      # of new alerts that started firing for that group.
      group_interval: 5m
    
      # If an alert has successfully been sent, wait 'repeat_interval' to
      # resend them.
      repeat_interval: 30m
    
      # A default receiver
      receiver: pam-alert
    
      # All the above attributes are inherited by all child routes and can
      # overwritten on each.
    
      # The child route trees.
      routes:
      - match:
          service: node
        receiver: pam-alert
    
        routes:
        - match:
            severity: critical
          receiver: pam-alert-critical
    
      # This route handles all alerts coming from a database service. If there's
      # no team to handle it, it defaults to the DB team.
      - match:
          service: database
        receiver: pam-alert
    
        routes:
        - match:
            severity: critical
          receiver: pam-alert-critical

      # This route handles all alerts coming from a database service. If there's
      # no team to handle it, it defaults to the DB team.
      - match:
          service: app
        receiver: pam-alert
    
        routes:
        - match:
            severity: critical
          receiver: pam-alert-critical

    receivers:
    - name: 'pam-alert'
      slack_configs:
      - api_url: <slackHookAlert>
        channel: '#pam-alert'
        send_resolved: true
        title: |-
          [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
          {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
            {{" "}}(
            {{- with .CommonLabels.Remove .GroupLabels.Names }}
              {{- range $index, $label := .SortedPairs -}}
                {{ if $index }}, {{ end }}
                {{- $label.Name }}="{{ $label.Value -}}"
              {{- end }}
            {{- end -}}
            )
          {{- end }}
        text: >-
          {{ range .Alerts -}}
          *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
      
          *Description:* {{ .Annotations.description }}
      
          *Details:*
            {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
            {{ end }}
          {{ end }}
    
    - name: 'pam-alert-critical'
      slack_configs:
      - api_url: <slackHookCritical>
        channel: '#pam-alert-criticals'
        send_resolved: true
        title: |-
          [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
          {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
            {{" "}}(
            {{- with .CommonLabels.Remove .GroupLabels.Names }}
              {{- range $index, $label := .SortedPairs -}}
                {{ if $index }}, {{ end }}
                {{- $label.Name }}="{{ $label.Value -}}"
              {{- end }}
            {{- end -}}
            )
          {{- end }}
        text: >-
          {{ range .Alerts -}}
          *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
      
          *Description:* {{ .Annotations.description }}
      
          *Details:*
            {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
            {{ end }}
          {{ end }}