UTF8: Content negotiation (encoding side)

config/http_config.go

@@ -30,7 +30,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mwitkow/go-conntrack"
+	conntrack "github.com/mwitkow/go-conntrack"
 	"golang.org/x/net/http/httpproxy"
 	"golang.org/x/net/http2"
 	"golang.org/x/oauth2"

expfmt/decode.go

@@ -61,10 +61,15 @@ func ResponseFormat(h http.Header) Format {
 		return FmtProtoDelim
 
 	case textType:
-		if v, ok := params["version"]; ok && v != TextVersion {
+		if v, ok := params["version"]; ok {
+			if v == TextVersion_0_0_4 {
+				return FmtText_0_0_4
+			} else if v == TextVersion_1_0_0 {
+				return FmtText_1_0_0
+			}
 			return FmtUnknown
 		}
-		return FmtText
+		return FmtText_0_0_4
 	}
 
 	return FmtUnknown
@@ -73,8 +78,8 @@ func ResponseFormat(h http.Header) Format {
 // NewDecoder returns a new decoder based on the given input format.
 // If the input format does not imply otherwise, a text format decoder is returned.
 func NewDecoder(r io.Reader, format Format) Decoder {
-	switch format {
-	case FmtProtoDelim:
+	switch format.FormatType() {
+	case TypeProtoDelim:
 		return &protoDecoder{r: r}
 	}
 	return &textDecoder{r: r}

expfmt/decode_test.go

@@ -17,6 +17,7 @@ import (
 	"bufio"
 	"errors"
 	"io"
+	"math"
 	"net/http"
 	"reflect"
 	"sort"
@@ -104,9 +105,10 @@ func TestProtoDecoder(t *testing.T) {
 	testTime := model.Now()
 
 	scenarios := []struct {
-		in       string
-		expected model.Vector
-		fail     bool
+		in             string
+		expected       model.Vector
+		legacyNameFail bool
+		fail           bool
 	}{
 		{
 			in: "",
@@ -332,6 +334,30 @@ func TestProtoDecoder(t *testing.T) {
 				},
 			},
 		},
+		{
+			in:             "\xa8\x01\n\ngauge.name\x12\x11gauge\ndoc\nstr\"ing\x18\x01\"T\n\x1b\n\x06name.1\x12\x11val with\nnew line\n*\n\x06name*2\x12 val with \\backslash and \"quotes\"\x12\t\t\x00\x00\x00\x00\x00\x00\xf0\x7f\"/\n\x10\n\x06name.1\x12\x06Björn\n\x10\n\x06name*2\x12\x06佖佥\x12\t\t\xd1\xcfD\xb9\xd0\x05\xc2H",
+			legacyNameFail: true,
+			expected: model.Vector{
+				&model.Sample{
+					Metric: model.Metric{
+						model.MetricNameLabel: "gauge.name",
+						"name.1":              "val with\nnew line",
+						"name*2":              "val with \\backslash and \"quotes\"",
+					},
+					Value:     model.SampleValue(math.Inf(+1)),
+					Timestamp: testTime,
+				},
+				&model.Sample{
+					Metric: model.Metric{
+						model.MetricNameLabel: "gauge.name",
+						"name.1":              "Björn",
+						"name*2":              "佖佥",
+					},
+					Value:     3.14e42,
+					Timestamp: testTime,
+				},
+			},
+		},
 	}
 
 	for i, scenario := range scenarios {
@@ -344,11 +370,31 @@ func TestProtoDecoder(t *testing.T) {
 
 		var all model.Vector
 		for {
+			model.NameValidationScheme = model.LegacyValidation
 			var smpls model.Vector
 			err := dec.Decode(&smpls)
 			if err != nil && errors.Is(err, io.EOF) {
 				break
 			}
+			if scenario.legacyNameFail {
+				if err == nil {
+					t.Fatal("Expected error when decoding without UTF-8 support enabled but got none")
+				}
+				model.NameValidationScheme = model.UTF8Validation
+				dec = &SampleDecoder{
+					Dec: &protoDecoder{r: strings.NewReader(scenario.in)},
+					Opts: &DecodeOptions{
+						Timestamp: testTime,
+					},
+				}
+				err = dec.Decode(&smpls)
+				if errors.Is(err, io.EOF) {
+					break
+				}
+				if err != nil {
+					t.Fatalf("Unexpected error when decoding with UTF-8 support: %v", err)
+				}
+			}
 			if scenario.fail {
 				if err == nil {
 					t.Fatal("Expected error but got none")
@@ -385,13 +431,17 @@ func testDiscriminatorHTTPHeader(t testing.TB) {
 			input:  map[string]string{"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="illegal"`},
 			output: FmtUnknown,
 		},
+		{
+			input:  map[string]string{"Content-Type": `text/plain; version=1.0.0`},
+			output: FmtText_1_0_0,
+		},
 		{
 			input:  map[string]string{"Content-Type": `text/plain; version=0.0.4`},
-			output: FmtText,
+			output: FmtText_0_0_4,
 		},
 		{
 			input:  map[string]string{"Content-Type": `text/plain`},
-			output: FmtText,
+			output: FmtText_0_0_4,
 		},
 		{
 			input:  map[string]string{"Content-Type": `text/plain; version=0.0.3`},
@@ -501,7 +551,7 @@ func TestTextDecoderWithBufioReader(t *testing.T) {
 
 	var decoded bool
 	r := bufio.NewReader(strings.NewReader(example))
-	dec := NewDecoder(r, FmtText)
+	dec := NewDecoder(r, FmtText_0_0_4)
 	for {
 		var mf dto.MetricFamily
 		if err := dec.Decode(&mf); err != nil {

expfmt/encode.go

@@ -21,6 +21,8 @@ import (
 	"google.golang.org/protobuf/encoding/protodelim"
 	"google.golang.org/protobuf/encoding/prototext"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg"
 
 	dto "github.com/prometheus/client_model/go"
@@ -64,20 +66,31 @@ func Negotiate(h http.Header) Format {
 	for _, ac := range goautoneg.ParseAccept(h.Get(hdrAccept)) {
 		ver := ac.Params["version"]
 		if ac.Type+"/"+ac.SubType == ProtoType && ac.Params["proto"] == ProtoProtocol {
+			utf8Suffix := Format("")
+			if ac.Params["validchars"] == UTF8Valid {
+				utf8Suffix = FmtUTF8Param
+			}
+
 			switch ac.Params["encoding"] {
 			case "delimited":
-				return FmtProtoDelim
+				return FmtProtoDelim + utf8Suffix
 			case "text":
-				return FmtProtoText
+				return FmtProtoText + utf8Suffix
 			case "compact-text":
-				return FmtProtoCompact
+				return FmtProtoCompact + utf8Suffix
 			}
 		}
-		if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion || ver == "") {
-			return FmtText
+		if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion_0_0_4 || ver == TextVersion_1_0_0 || ver == "") {
+			if ver == TextVersion_1_0_0 {
+				if ac.Params["validchars"] == UTF8Valid {
+					return FmtText_1_0_0 + FmtUTF8Param
+				}
+				return FmtText_1_0_0
+			}
+			return FmtText_0_0_4
 		}
 	}
-	return FmtText
+	return FmtText_0_0_4
 }
 
 // NegotiateIncludingOpenMetrics works like Negotiate but includes
@@ -88,26 +101,44 @@ func NegotiateIncludingOpenMetrics(h http.Header) Format {
 	for _, ac := range goautoneg.ParseAccept(h.Get(hdrAccept)) {
 		ver := ac.Params["version"]
 		if ac.Type+"/"+ac.SubType == ProtoType && ac.Params["proto"] == ProtoProtocol {
+			utf8Suffix := Format("")
+			if ac.Params["validchars"] == UTF8Valid {
+				utf8Suffix = FmtUTF8Param
+			}
+
 			switch ac.Params["encoding"] {
 			case "delimited":
-				return FmtProtoDelim
+				return FmtProtoDelim + utf8Suffix
 			case "text":
-				return FmtProtoText
+				return FmtProtoText + utf8Suffix
 			case "compact-text":
-				return FmtProtoCompact
+				return FmtProtoCompact + utf8Suffix
+			}
+		}
+		if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion_1_0_0 || ver == "") {
+			if ac.Params["validchars"] == UTF8Valid {
+				return FmtText_1_0_0 + FmtUTF8Param
 			}
+			return FmtText_0_0_4
 		}
-		if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion || ver == "") {
-			return FmtText
+		if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion_0_0_4 || ver == "") {
+			return FmtText_0_0_4
 		}
-		if ac.Type+"/"+ac.SubType == OpenMetricsType && (ver == OpenMetricsVersion_0_0_1 || ver == OpenMetricsVersion_1_0_0 || ver == "") {
-			if ver == OpenMetricsVersion_1_0_0 {
+		if ac.Type+"/"+ac.SubType == OpenMetricsType && (ver == OpenMetricsVersion_0_0_1 || ver == OpenMetricsVersion_1_0_0 || ver == OpenMetricsVersion_2_0_0 || ver == "") {
+			switch ver {
+			case OpenMetricsVersion_2_0_0:
+				if ac.Params["validchars"] == UTF8Valid {
+					return FmtOpenMetrics_2_0_0 + FmtUTF8Param
+				}
+				return FmtOpenMetrics_2_0_0
+			case OpenMetricsVersion_1_0_0:
 				return FmtOpenMetrics_1_0_0
+			default:
+				return FmtOpenMetrics_0_0_1
 			}
-			return FmtOpenMetrics_0_0_1
 		}
 	}
-	return FmtText
+	return FmtText_0_0_4
 }
 
 // NewEncoder returns a new encoder based on content type negotiation. All
@@ -116,44 +147,40 @@ func NegotiateIncludingOpenMetrics(h http.Header) Format {
 // for FmtOpenMetrics, but a future (breaking) release will add the Close method
 // to the Encoder interface directly. The current version of the Encoder
 // interface is kept for backwards compatibility.
+// In cases where the Format does not allow for UTF8 names, the global
+// NameEscapingScheme will be applied.
 func NewEncoder(w io.Writer, format Format) Encoder {
+	escapingScheme := format.ToEscapingScheme()
+
 	switch format {
 	case FmtProtoDelim:
 		return encoderCloser{
 			encode: func(v *dto.MetricFamily) error {
-				_, err := protodelim.MarshalTo(w, v)
-				return err
-			},
-			close: func() error { return nil },
-		}
-	case FmtProtoCompact:
-		return encoderCloser{
-			encode: func(v *dto.MetricFamily) error {
-				_, err := fmt.Fprintln(w, v.String())
+				_, err := fmt.Fprintln(w, model.EscapeMetricFamily(v, escapingScheme).String())
 				return err
 			},
 			close: func() error { return nil },
 		}
-	case FmtProtoText:
+	case TypeProtoText:
 		return encoderCloser{
 			encode: func(v *dto.MetricFamily) error {
-				_, err := fmt.Fprintln(w, prototext.Format(v))
+				_, err := fmt.Fprintln(w, prototext.Format(model.EscapeMetricFamily(v, escapingScheme)))
 				return err
 			},
 			close: func() error { return nil },
 		}
-	case FmtText:
+	case TypeTextPlain:
 		return encoderCloser{
 			encode: func(v *dto.MetricFamily) error {
-				_, err := MetricFamilyToText(w, v)
+				_, err := MetricFamilyToText(w, model.EscapeMetricFamily(v, escapingScheme))
 				return err
 			},
 			close: func() error { return nil },
 		}
-	case FmtOpenMetrics_0_0_1, FmtOpenMetrics_1_0_0:
+	case TypeOpenMetrics:
 		return encoderCloser{
 			encode: func(v *dto.MetricFamily) error {
-				_, err := MetricFamilyToOpenMetrics(w, v)
+				_, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme))
 				return err
 			},
 			close: func() error {