Skip to content

Commit 300a9ed

Browse files
nicolastakashisaswatamcodematej-g
authored
[FEATURE] adding otlp endpoint (#7996)
* [FEATURE] adding otlp endpoint Signed-off-by: Nicolas Takashi <[email protected]> * [FEATURE] adding otlp endpoint Signed-off-by: Nicolas Takashi <[email protected]> * [FEATURE] adding otlp endpoint Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] e2e tests for otlp receiver Signed-off-by: Nicolas Takashi <[email protected]> * [CHORE] adding otlp flags Signed-off-by: Nicolas Takashi <[email protected]> * [DOC] updating docs Signed-off-by: Nicolas Takashi <[email protected]> * [CHORE] copying otlptranslator Signed-off-by: Nicolas Takashi <[email protected]> * [CHORE] copying otlptranslator tests Signed-off-by: Nicolas Takashi <[email protected]> * [CHORE] copying otlptranslator tests Signed-off-by: Nicolas Takashi <[email protected]> * [CHORE] copying otlptranslator tests Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] lint issues Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] lint issues Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] lint issues Signed-off-by: Nicolas Takashi <[email protected]> * [CHORE] using multi errors Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] span naming convention Signed-off-by: Nicolas Takashi <[email protected]> * [TEST] adding handler otlp unit test Signed-off-by: Nicolas Takashi <[email protected]> * [TEST] upgrade collector version Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] golang lint Signed-off-by: Nicolas Takashi <[email protected]> * [CHORE] adding allow size bytes limit gate Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] unit test otlp endpoint Signed-off-by: Nicolas Takashi <[email protected]> * Apply suggestions from code review Co-authored-by: Saswata Mukherjee <[email protected]> Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] unit test otlp endpoint Signed-off-by: Nicolas Takashi <[email protected]> * [DOC] updating docs Signed-off-by: Nicolas Takashi <[email protected]> * [CHORE] applying pr comments Signed-off-by: Nicolas Takashi <[email protected]> * Update pkg/receive/handler_otlp.go Co-authored-by: Matej Gera <[email protected]> Signed-off-by: Nicolas Takashi <[email protected]> * Update cmd/thanos/receive.go Co-authored-by: Matej Gera <[email protected]> Signed-off-by: Nicolas Takashi <[email protected]> * [DOCS] updating Signed-off-by: Nicolas Takashi <[email protected]> * [FIX] go mod lint issues Signed-off-by: Nicolas Takashi <[email protected]> * Fix TestFromMetrics error comparison Signed-off-by: Saswata Mukherjee <[email protected]> --------- Signed-off-by: Nicolas Takashi <[email protected]> Signed-off-by: Saswata Mukherjee <[email protected]> Co-authored-by: Saswata Mukherjee <[email protected]> Co-authored-by: Matej Gera <[email protected]>
1 parent a3b78c2 commit 300a9ed

25 files changed

+4085
-39
lines changed

Diff for: ‎cmd/thanos/receive.go

+7
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,8 @@ func runReceive(
290290

291291
AsyncForwardWorkerCount: conf.asyncForwardWorkerCount,
292292
ReplicationProtocol: receive.ReplicationProtocol(conf.replicationProtocol),
293+
OtlpEnableTargetInfo: conf.otlpEnableTargetInfo,
294+
OtlpResourceAttributes: conf.otlpResourceAttributes,
293295
})
294296

295297
grpcProbe := prober.NewGRPC()
@@ -909,6 +911,8 @@ type receiveConfig struct {
909911

910912
headExpandedPostingsCacheSize uint64
911913
compactedBlocksExpandedPostingsCacheSize uint64
914+
otlpEnableTargetInfo bool
915+
otlpResourceAttributes []string
912916
}
913917

914918
func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {
@@ -1066,6 +1070,9 @@ func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {
10661070
cmd.Flag("receive.limits-config-reload-timer", "Minimum amount of time to pass for the limit configuration to be reloaded. Helps to avoid excessive reloads.").
10671071
Default("1s").Hidden().DurationVar(&rc.limitsConfigReloadTimer)
10681072

1073+
cmd.Flag("receive.otlp-enable-target-info", "Enables target information in OTLP metrics ingested by Receive. If enabled, it converts the resource to the target info metric").Default("true").BoolVar(&rc.otlpEnableTargetInfo)
1074+
cmd.Flag("receive.otlp-promote-resource-attributes", "(Repeatable) Resource attributes to include in OTLP metrics ingested by Receive.").Default("").StringsVar(&rc.otlpResourceAttributes)
1075+
10691076
rc.featureList = cmd.Flag("enable-feature", "Comma separated experimental feature names to enable. The current list of features is "+metricNamesFilter+".").Default("").Strings()
10701077
}
10711078

Diff for: ‎docs/components/receive.md

+8-1
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ Please see the metric `thanos_receive_forward_delay_seconds` to see if you need
331331

332332
The following formula is used for calculating quorum:
333333

334-
```go mdox-exec="sed -n '1015,1025p' pkg/receive/handler.go"
334+
```go mdox-exec="sed -n '1029,1039p' pkg/receive/handler.go"
335335
// writeQuorum returns minimum number of replicas that has to confirm write success before claiming replication success.
336336
func (h *Handler) writeQuorum() int {
337337
// NOTE(GiedriusS): this is here because otherwise RF=2 doesn't make sense as all writes
@@ -458,6 +458,13 @@ Flags:
458458
configuration. If it's empty AND hashring
459459
configuration was provided, it means that
460460
receive will run in RoutingOnly mode.
461+
--receive.otlp-enable-target-info
462+
Enables target information in OTLP metrics
463+
ingested by Receive. If enabled, it converts
464+
the resource to the target info metric
465+
--receive.otlp-promote-resource-attributes= ...
466+
(Repeatable) Resource attributes to include in
467+
OTLP metrics ingested by Receive.
461468
--receive.relabel-config=<content>
462469
Alternative to 'receive.relabel-config-file'
463470
flag (mutually exclusive). Content of YAML file

Diff for: ‎go.mod

+48-38
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,55 @@ require (
114114
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
115115
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 // indirect
116116
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
117-
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.32.3 // indirect
117+
)
118+
119+
require (
120+
go.opentelemetry.io/collector/pdata v1.14.1
121+
go.opentelemetry.io/collector/semconv v0.108.1
122+
)
123+
124+
require github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect
125+
126+
require (
118127
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
128+
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
129+
github.com/cilium/ebpf v0.11.0 // indirect
130+
github.com/containerd/cgroups/v3 v3.0.3 // indirect
131+
github.com/docker/go-units v0.5.0 // indirect
132+
github.com/elastic/go-licenser v0.3.1 // indirect
133+
github.com/go-ini/ini v1.67.0 // indirect
134+
github.com/go-openapi/runtime v0.27.1 // indirect
135+
github.com/goccy/go-json v0.10.3 // indirect
136+
github.com/godbus/dbus/v5 v5.0.4 // indirect
137+
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
138+
github.com/google/s2a-go v0.1.8 // indirect
139+
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible // indirect
140+
github.com/jcchavezs/porto v0.1.0 // indirect
141+
github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 // indirect
142+
github.com/mdlayher/socket v0.4.1 // indirect
143+
github.com/mdlayher/vsock v1.2.1 // indirect
144+
github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect
145+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
146+
github.com/onsi/ginkgo v1.16.5 // indirect
147+
github.com/opencontainers/runtime-spec v1.0.2 // indirect
148+
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
149+
github.com/sercand/kuberesolver/v4 v4.0.0 // indirect
150+
github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39 // indirect
151+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
152+
go.opentelemetry.io/contrib/propagators/ot v1.29.0 // indirect
153+
go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect
154+
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
155+
google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect
156+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
157+
k8s.io/apimachinery v0.31.1 // indirect
158+
k8s.io/client-go v0.31.0 // indirect
159+
k8s.io/klog/v2 v2.130.1 // indirect
160+
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
161+
zenhack.net/go/util v0.0.0-20230414204917-531d38494cf5 // indirect
162+
)
163+
164+
require (
165+
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.32.3 // indirect
119166
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
120167
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
121168
github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible // indirect
@@ -134,24 +181,17 @@ require (
134181
github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect
135182
github.com/aws/smithy-go v1.11.1 // indirect
136183
github.com/baidubce/bce-sdk-go v0.9.111 // indirect
137-
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
138184
github.com/beorn7/perks v1.0.1 // indirect
139185
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
140186
github.com/chromedp/sysutil v1.0.0 // indirect
141-
github.com/cilium/ebpf v0.11.0 // indirect
142187
github.com/clbanning/mxj v1.8.4 // indirect
143-
github.com/containerd/cgroups/v3 v3.0.3 // indirect
144188
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
145189
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
146190
github.com/dennwc/varint v1.0.0 // indirect
147-
github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect
148-
github.com/docker/go-units v0.5.0 // indirect
149191
github.com/edsrzf/mmap-go v1.1.0 // indirect
150-
github.com/elastic/go-licenser v0.3.1 // indirect
151192
github.com/elastic/go-sysinfo v1.8.1 // indirect
152193
github.com/elastic/go-windows v1.0.1 // indirect
153194
github.com/felixge/httpsnoop v1.0.4 // indirect
154-
github.com/go-ini/ini v1.67.0 // indirect
155195
github.com/go-logfmt/logfmt v0.6.0 // indirect
156196
github.com/go-logr/logr v1.4.2 // indirect
157197
github.com/go-logr/stdr v1.2.2 // indirect
@@ -161,56 +201,41 @@ require (
161201
github.com/go-openapi/jsonpointer v0.20.2 // indirect
162202
github.com/go-openapi/jsonreference v0.20.4 // indirect
163203
github.com/go-openapi/loads v0.21.5 // indirect
164-
github.com/go-openapi/runtime v0.27.1 // indirect
165204
github.com/go-openapi/spec v0.20.14 // indirect
166205
github.com/go-openapi/swag v0.22.9 // indirect
167206
github.com/go-openapi/validate v0.23.0 // indirect
168207
github.com/gobwas/httphead v0.1.0 // indirect
169208
github.com/gobwas/pool v0.2.1 // indirect
170209
github.com/gobwas/ws v1.2.1 // indirect
171-
github.com/goccy/go-json v0.10.3 // indirect
172-
github.com/godbus/dbus/v5 v5.0.4 // indirect
173210
github.com/gofrs/flock v0.8.1 // indirect
174211
github.com/gogo/googleapis v1.4.0 // indirect
175-
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
176212
github.com/google/go-querystring v1.1.0 // indirect
177213
github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
178-
github.com/google/s2a-go v0.1.8 // indirect
179214
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
180215
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
181216
github.com/gorilla/mux v1.8.0 // indirect
182217
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
183218
github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
184-
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible // indirect
185-
github.com/jcchavezs/porto v0.1.0 // indirect
186219
github.com/jmespath/go-jmespath v0.4.0 // indirect
187220
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
188221
github.com/josharian/intern v1.0.0 // indirect
189222
github.com/julienschmidt/httprouter v1.3.0 // indirect
190223
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
191224
github.com/kylelemons/godebug v1.1.0 // indirect
192-
github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 // indirect
193225
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 // indirect
194226
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
195227
github.com/mailru/easyjson v0.7.7 // indirect
196228
github.com/mattn/go-runewidth v0.0.13 // indirect
197-
github.com/mdlayher/socket v0.4.1 // indirect
198-
github.com/mdlayher/vsock v1.2.1 // indirect
199-
github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect
200229
github.com/minio/md5-simd v1.1.2 // indirect
201230
github.com/minio/minio-go/v7 v7.0.80 // indirect
202231
github.com/mitchellh/mapstructure v1.5.0 // indirect
203232
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
204233
github.com/modern-go/reflect2 v1.0.2 // indirect
205234
github.com/mozillazg/go-httpheader v0.2.1 // indirect
206-
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
207235
github.com/ncw/swift v1.0.53 // indirect
208-
github.com/onsi/ginkgo v1.16.5 // indirect
209-
github.com/opencontainers/runtime-spec v1.0.2 // indirect
210236
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
211237
github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
212238
github.com/oracle/oci-go-sdk/v65 v65.41.1 // indirect
213-
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
214239
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
215240
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
216241
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
@@ -219,7 +244,6 @@ require (
219244
github.com/rivo/uniseg v0.2.0 // indirect
220245
github.com/rs/xid v1.6.0 // indirect
221246
github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect
222-
github.com/sercand/kuberesolver/v4 v4.0.0 // indirect
223247
github.com/shirou/gopsutil/v3 v3.22.9 // indirect
224248
github.com/sirupsen/logrus v1.9.3 // indirect
225249
github.com/stretchr/objx v0.5.2 // indirect
@@ -230,40 +254,26 @@ require (
230254
github.com/weaveworks/promrus v1.2.0 // indirect
231255
github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 // indirect
232256
github.com/yusufpapurcu/wmi v1.2.2 // indirect
233-
github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39 // indirect
234257
go.elastic.co/apm/module/apmhttp v1.15.0 // indirect
235258
go.elastic.co/fastjson v1.1.0 // indirect
236259
go.mongodb.org/mongo-driver v1.14.0 // indirect
237260
go.opencensus.io v0.24.0 // indirect
238-
go.opentelemetry.io/collector/pdata v1.14.1 // indirect
239-
go.opentelemetry.io/collector/semconv v0.108.1 // indirect
240-
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
241261
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
242262
go.opentelemetry.io/contrib/propagators/aws v1.29.0 // indirect
243263
go.opentelemetry.io/contrib/propagators/b3 v1.29.0 // indirect
244264
go.opentelemetry.io/contrib/propagators/jaeger v1.29.0 // indirect
245-
go.opentelemetry.io/contrib/propagators/ot v1.29.0 // indirect
246265
go.opentelemetry.io/otel/metric v1.31.0 // indirect
247266
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
248267
go.uber.org/multierr v1.11.0 // indirect
249-
go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect
250268
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
251-
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
252269
golang.org/x/mod v0.21.0 // indirect
253270
golang.org/x/oauth2 v0.23.0 // indirect
254271
golang.org/x/sys v0.28.0 // indirect
255272
golang.org/x/tools v0.24.0 // indirect
256273
gonum.org/v1/gonum v0.15.0 // indirect
257274
google.golang.org/api v0.195.0 // indirect
258275
google.golang.org/genproto v0.0.0-20240823204242-4ba0660f739c // indirect
259-
google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect
260-
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
261276
howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
262-
k8s.io/apimachinery v0.31.1 // indirect
263-
k8s.io/client-go v0.31.0 // indirect
264-
k8s.io/klog/v2 v2.130.1 // indirect
265-
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
266-
zenhack.net/go/util v0.0.0-20230414204917-531d38494cf5 // indirect
267277
)
268278

269279
replace (

Diff for: ‎pkg/errutil/multierror.go

+9
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,15 @@ func (es NonNilMultiError) Cause() error {
7171
return es.getCause()
7272
}
7373

74+
func (es NonNilMultiError) Is(target error) bool {
75+
for _, err := range es {
76+
if errors.Is(err, target) {
77+
return true
78+
}
79+
}
80+
return false
81+
}
82+
7483
func (es NonNilMultiError) getCause() NonNilMultiRootError {
7584
var causes []error
7685
for _, err := range es {

Diff for: ‎pkg/receive/handler.go

+14
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ type Options struct {
116116
Limiter *Limiter
117117
AsyncForwardWorkerCount uint
118118
ReplicationProtocol ReplicationProtocol
119+
OtlpEnableTargetInfo bool
120+
OtlpResourceAttributes []string
119121
}
120122

121123
// Handler serves a Prometheus remote write receiving HTTP endpoint.
@@ -275,6 +277,18 @@ func NewHandler(logger log.Logger, o *Options) *Handler {
275277
),
276278
)
277279

280+
h.router.Post(
281+
"/api/v1/otlp",
282+
instrf(
283+
"otlp",
284+
readyf(
285+
middleware.RequestID(
286+
http.HandlerFunc(h.receiveOTLPHTTP),
287+
),
288+
),
289+
),
290+
)
291+
278292
statusAPI := statusapi.New(statusapi.Options{
279293
GetStats: h.getStats,
280294
Registry: h.options.Registry,

0 commit comments

Comments
 (0)