diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go
index 5a7f21d3ba0..0332c33eaa3 100644
--- a/cmd/promtool/main.go
+++ b/cmd/promtool/main.go
@@ -210,6 +210,7 @@ func main() {
"test-rule-file",
"The unit test file.",
).Required().ExistingFiles()
+ testRulesDiff := testRulesCmd.Flag("diff", "[Experimental] Print colored differential output between expected & received output.").Default("false").Bool()
defaultDBPath := "data/"
tsdbCmd := app.Command("tsdb", "Run tsdb commands.")
@@ -375,6 +376,7 @@ func main() {
EnableNegativeOffset: true,
},
*testRulesRun,
+ *testRulesDiff,
*testRulesFiles...),
)
diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go
index a25a8596d42..a89288c44a4 100644
--- a/cmd/promtool/unittest.go
+++ b/cmd/promtool/unittest.go
@@ -15,6 +15,7 @@ package main
import (
"context"
+ "encoding/json"
"errors"
"fmt"
"os"
@@ -27,6 +28,7 @@ import (
"github.com/go-kit/log"
"github.com/grafana/regexp"
+ "github.com/nsf/jsondiff"
"github.com/prometheus/common/model"
"gopkg.in/yaml.v2"
@@ -40,7 +42,7 @@ import (
// RulesUnitTest does unit testing of rules based on the unit testing files provided.
// More info about the file format can be found in the docs.
-func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files ...string) int {
+func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
failed := false
var run *regexp.Regexp
@@ -49,7 +51,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files .
}
for _, f := range files {
- if errs := ruleUnitTest(f, queryOpts, run); errs != nil {
+ if errs := ruleUnitTest(f, queryOpts, run, diffFlag); errs != nil {
fmt.Fprintln(os.Stderr, " FAILED:")
for _, e := range errs {
fmt.Fprintln(os.Stderr, e.Error())
@@ -67,7 +69,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files .
return successExitCode
}
-func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp) []error {
+func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
fmt.Println("Unit Testing: ", filename)
b, err := os.ReadFile(filename)
@@ -109,7 +111,7 @@ func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.
if t.Interval == 0 {
t.Interval = unitTestInp.EvaluationInterval
}
- ers := t.test(evalInterval, groupOrderMap, queryOpts, unitTestInp.RuleFiles...)
+ ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...)
if ers != nil {
errs = append(errs, ers...)
}
@@ -173,7 +175,7 @@ type testGroup struct {
}
// test performs the unit tests.
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, ruleFiles ...string) []error {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) []error {
// Setup testing suite.
suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString(), queryOpts)
if err != nil {
@@ -345,8 +347,44 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
}
expString := indentLines(expAlerts.String(), " ")
gotString := indentLines(gotAlerts.String(), " ")
- errs = append(errs, fmt.Errorf("%s alertname: %s, time: %s, \n exp:%v, \n got:%v",
- testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
+ if diffFlag {
+ // If empty, populates an empty value
+ if gotAlerts.Len() == 0 {
+ gotAlerts = append(gotAlerts, labelAndAnnotation{
+ Labels: labels.Labels{},
+ Annotations: labels.Labels{},
+ })
+ }
+ // If empty, populates an empty value
+ if expAlerts.Len() == 0 {
+ expAlerts = append(expAlerts, labelAndAnnotation{
+ Labels: labels.Labels{},
+ Annotations: labels.Labels{},
+ })
+ }
+
+ diffOpts := jsondiff.DefaultConsoleOptions()
+ expAlertsJSON, err := json.Marshal(expAlerts)
+ if err != nil {
+ errs = append(errs, fmt.Errorf("error marshaling expected %s alert: [%s]", tg.TestGroupName, err.Error()))
+ continue
+ }
+
+ gotAlertsJSON, err := json.Marshal(gotAlerts)
+ if err != nil {
+ errs = append(errs, fmt.Errorf("error marshaling received %s alert: [%s]", tg.TestGroupName, err.Error()))
+ continue
+ }
+
+ res, diff := jsondiff.Compare(expAlertsJSON, gotAlertsJSON, &diffOpts)
+ if res != jsondiff.FullMatch {
+ errs = append(errs, fmt.Errorf("%s alertname: %s, time: %s, \n diff: %v",
+ testName, testcase.Alertname, testcase.EvalTime.String(), indentLines(diff, " ")))
+ }
+ } else {
+ errs = append(errs, fmt.Errorf("%s alertname: %s, time: %s, \n exp:%v, \n got:%v",
+ testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
+ }
}
}
diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go
index fb4012e3c14..b8170d784e4 100644
--- a/cmd/promtool/unittest_test.go
+++ b/cmd/promtool/unittest_test.go
@@ -125,7 +125,7 @@ func TestRulesUnitTest(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- if got := RulesUnitTest(tt.queryOpts, nil, tt.args.files...); got != tt.want {
+ if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
})
@@ -178,7 +178,7 @@ func TestRulesUnitTestRun(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- if got := RulesUnitTest(tt.queryOpts, tt.args.run, tt.args.files...); got != tt.want {
+ if got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
})
diff --git a/docs/command-line/promtool.md b/docs/command-line/promtool.md
index ba948685b6d..863bc068c34 100644
--- a/docs/command-line/promtool.md
+++ b/docs/command-line/promtool.md
@@ -450,9 +450,10 @@ Unit tests for rules.
###### Flags
-| Flag | Description |
-| --- | --- |
-| --run | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. |
+| Flag | Description | Default |
+| --- | --- | --- |
+| --run | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. | |
+| --diff | [Experimental] Print colored differential output between expected & received output. | `false` |
diff --git a/go.mod b/go.mod
index c34d3db2eae..4833f9c1126 100644
--- a/go.mod
+++ b/go.mod
@@ -43,6 +43,7 @@ require (
github.com/miekg/dns v1.1.57
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
+ github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1
github.com/oklog/run v1.1.0
github.com/oklog/ulid v1.3.1
github.com/ovh/go-ovh v1.4.3
diff --git a/go.sum b/go.sum
index 3593e39da63..daac5e3203d 100644
--- a/go.sum
+++ b/go.sum
@@ -579,6 +579,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM=
+github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=