feat: add configurable namespace label to prometheus metrics (#145)

Author: damiankaminski-form3

internal/run/main_test.go

@@ -12,6 +12,8 @@ import (
 
 var fakePrometheus FakePrometheus
 
+const fakePrometheusNamespace = "test-namespace"
+
 func TestMain(m *testing.M) {
 
 	var err error
@@ -23,6 +25,10 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		log.Fatal(err)
 	}
+	err = os.Setenv("PROMETHEUS_NAMESPACE", fakePrometheusNamespace)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	fakePrometheus.StartServer()
 

internal/run/prom_push_gw_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -24,12 +25,43 @@ type FakePrometheus struct {
 }
 
 func (f *FakePrometheus) ServeHTTP(response http.ResponseWriter, request *http.Request) {
+	parseGroupedLabels := func() []*io_prometheus_client.LabelPair {
+		// labels added via push.Grouping are passed through URI
+		// example: /metrics/job/f1-f94b1fd3-1a08-4829-896e-792397ccdbfd/namespace/test-namespace/abc/cde
+
+		var labels []*io_prometheus_client.LabelPair
+		parts := strings.Split(request.RequestURI, "/")[4:]
+
+		var labelName string
+		for i, part := range parts {
+			if i%2 == 0 {
+				labelName = part
+				continue
+			}
+
+			name := labelName
+			value := part
+			labels = append(labels, &io_prometheus_client.LabelPair{
+				Name:  &name,
+				Value: &value,
+			})
+		}
+
+		return labels
+	}
+
 	if request != nil && request.Body != nil {
 		defer errorh.SafeClose(request.Body)
 		metricFamily := &io_prometheus_client.MetricFamily{}
 		expfmt.NewDecoder(request.Body, expfmt.ResponseFormat(request.Header)).Decode(metricFamily)
 		mf, ok := f.metricFamilies.Load(*metricFamily.Name)
 		if !ok {
+			if metricFamily.Metric != nil {
+				groupedLabels := parseGroupedLabels()
+				for _, m := range metricFamily.Metric {
+					m.Label = append(m.Label, groupedLabels...)
+				}
+			}
 			f.metricFamilies.Store(*metricFamily.Name, metricFamily)
 		} else {
 			value, ok := mf.(*io_prometheus_client.MetricFamily)

internal/run/run_cmd_test.go

@@ -7,7 +7,7 @@ import (
 
 // TestSimpleFlow is equivalent to a single run from TestParameterised. It's useful for debugging individual test runs
 func TestSimpleFlow(t *testing.T) {
-	//t.Skip("Duplicate of Parameterised test. Useful for manual testing when adding new tests or debugging, so leaving in place")
+	// t.Skip("Duplicate of Parameterised test. Useful for manual testing when adding new tests or debugging, so leaving in place")
 	given, when, then := NewRunTestStage(t)
 
 	test := TestParam{
@@ -585,6 +585,20 @@ func TestSetupMetricsAreRecorded(t *testing.T) {
 		there_is_a_metric_called("form3_loadtest_setup")
 }
 
+func TestNamespaceLabel(t *testing.T) {
+	given, when, then := NewRunTestStage(t)
+
+	given.
+		a_rate_of("10/s").and().
+		a_scenario_where_each_iteration_takes(1 * time.Millisecond)
+
+	when.i_execute_the_run_command()
+
+	then.
+		metrics_are_pushed_to_prometheus().and().
+		all_exported_metrics_contain_label("namespace", fakePrometheusNamespace)
+}
+
 func TestFailureCounts(t *testing.T) {
 	given, when, then := NewRunTestStage(t)
 

internal/run/run_stage_test.go

@@ -3,16 +3,17 @@ package run_test
 import (
 	"fmt"
 	"math"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"testing"
 	"time"
 
 	"github.com/form3tech-oss/f1/v2/internal/run"
-	"github.com/form3tech-oss/f1/v2/pkg/f1"
-
 	"github.com/form3tech-oss/f1/v2/internal/trigger/ramp"
+	"github.com/form3tech-oss/f1/v2/pkg/f1"
+	"github.com/prometheus/common/expfmt"
 
 	"github.com/form3tech-oss/f1/v2/internal/trigger/file"
 
@@ -553,6 +554,32 @@ func (s *RunTestStage) the_iteration_metric_has_n_results(n int, result string)
 	return s
 }
 
+func (s *RunTestStage) all_exported_metrics_contain_label(labelName string, labelValue string) *RunTestStage {
+	metricNames := fakePrometheus.GetMetricNames()
+
+	for _, name := range metricNames {
+		metricFamily := fakePrometheus.GetMetricFamily(name)
+		s.require.NotNil(metricFamily)
+
+		for _, metric := range metricFamily.Metric {
+			match := false
+			for _, label := range metric.Label {
+				nameMatch := label.GetName() == labelName
+				valueMatch := label.GetValue() == labelValue
+				match = match || (nameMatch && valueMatch)
+			}
+
+			if !match {
+				openMetrics := strings.Builder{}
+				_, _ = expfmt.MetricFamilyToOpenMetrics(&openMetrics, metricFamily)
+				s.require.FailNowf("Label is missing", "Metric %q do not have label %q with value %q:\n%s",
+					metricFamily.GetName(), labelName, labelValue, openMetrics.String())
+			}
+		}
+	}
+	return s
+}
+
 func getMetricByResult(metricFamily *io_prometheus_client.MetricFamily, result string) *io_prometheus_client.Metric {
 	for _, metric := range metricFamily.Metric {
 		for _, label := range metric.Label {

internal/run/test_runner.go

@@ -43,6 +43,11 @@ func NewRun(options options.RunOptions, t *api.Trigger) (*Run, error) {
 	prometheusUrl := os.Getenv("PROMETHEUS_PUSH_GATEWAY")
 	if prometheusUrl != "" {
 		run.pusher = push.New(prometheusUrl, "f1-"+options.Scenario).Gatherer(prometheus.DefaultGatherer)
+
+		prometheusNamespace := os.Getenv("PROMETHEUS_NAMESPACE")
+		if prometheusNamespace != "" {
+			run.pusher = run.pusher.Grouping("namespace", prometheusNamespace)
+		}
 	}
 	if run.Options.RegisterLogHookFunc == nil {
 		run.Options.RegisterLogHookFunc = logging.NoneRegisterLogHookFunc