diff --git a/.chloggen/memorylimiter_middleware.yaml b/.chloggen/memorylimiter_middleware.yaml new file mode 100644 index 00000000000..a886e95c957 --- /dev/null +++ b/.chloggen/memorylimiter_middleware.yaml @@ -0,0 +1,25 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: extension/memory_limiter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: The memorylimiter extension can be used as an HTTP/GRPC middleware. + +# One or more tracking issues or pull requests related to the change +issues: [14081] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/cmd/otelcorecol/go.mod b/cmd/otelcorecol/go.mod index 3eb6c73255c..044586678e1 100644 --- a/cmd/otelcorecol/go.mod +++ b/cmd/otelcorecol/go.mod @@ -112,7 +112,7 @@ require ( go.opentelemetry.io/collector/exporter/xexporter v0.139.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.45.0 // indirect go.opentelemetry.io/collector/extension/extensioncapabilities v0.139.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.139.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.45.0 // indirect go.opentelemetry.io/collector/extension/extensiontest v0.139.0 // indirect go.opentelemetry.io/collector/extension/xextension v0.139.0 // indirect go.opentelemetry.io/collector/featuregate v1.45.0 // indirect diff --git a/extension/memorylimiterextension/README.md b/extension/memorylimiterextension/README.md index a54f0fea3db..066f43465a1 100644 --- a/extension/memorylimiterextension/README.md +++ b/extension/memorylimiterextension/README.md @@ -14,6 +14,29 @@ The memory limiter extension is used to prevent out of memory situations on the collector. The extension will potentially replace the Memory Limiter Processor. It provides better guarantees from running out of memory as it will be used by the receivers to reject requests before converting them into OTLP. All the configurations -are the same as Memory Limiter Processor. The extension is under development and does nothing. +are the same as Memory Limiter Processor. + + +This extension can be used as an extension for all HTTP and gRPC receivers that +are configured through the standard `confighttp` and `configgrpc` libraries. For +example, to configure this extension in the OTLP receiver: + +``` +receivers: + otlp: + protocols: + grpc: + middlewares: + - id: memory_limiter + http: + middlewares: + - id: memory_limiter + +extensions: + memory_limiter: + check_interval: 1s + limit_percentage: 1 + spike_limit_percentage: 0.05 +``` see [memorylimiterprocessor](../../processor/memorylimiterprocessor/README.md) for additional details diff --git a/extension/memorylimiterextension/go.mod b/extension/memorylimiterextension/go.mod index a1da9a59cba..2a4a06cf5bd 100644 --- a/extension/memorylimiterextension/go.mod +++ b/extension/memorylimiterextension/go.mod @@ -8,10 +8,12 @@ require ( go.opentelemetry.io/collector/component/componenttest v0.139.0 go.opentelemetry.io/collector/confmap v1.45.0 go.opentelemetry.io/collector/extension v1.45.0 + go.opentelemetry.io/collector/extension/extensionmiddleware v1.45.0 go.opentelemetry.io/collector/extension/extensiontest v0.139.0 go.opentelemetry.io/collector/internal/memorylimiter v0.139.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 + google.golang.org/grpc v1.76.0 ) require ( @@ -49,7 +51,11 @@ require ( go.opentelemetry.io/otel/trace v1.38.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.42.0 // indirect golang.org/x/sys v0.37.0 // indirect + golang.org/x/text v0.27.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -70,3 +76,5 @@ replace go.opentelemetry.io/collector/extension/extensiontest => ../../extension replace go.opentelemetry.io/collector/featuregate => ../../featuregate replace go.opentelemetry.io/collector/internal/testutil => ../../internal/testutil + +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware diff --git a/extension/memorylimiterextension/go.sum b/extension/memorylimiterextension/go.sum index dc3464c4ef6..f099af7a1b2 100644 --- a/extension/memorylimiterextension/go.sum +++ b/extension/memorylimiterextension/go.sum @@ -14,6 +14,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -90,11 +92,21 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/extension/memorylimiterextension/memorylimiter.go b/extension/memorylimiterextension/memorylimiter.go index 4986243ce10..c2ebe0563c1 100644 --- a/extension/memorylimiterextension/memorylimiter.go +++ b/extension/memorylimiterextension/memorylimiter.go @@ -5,8 +5,12 @@ package memorylimiterextension // import "go.opentelemetry.io/collector/extensio import ( "context" + "net/http" "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/internal/memorylimiter" @@ -38,3 +42,24 @@ func (ml *memoryLimiterExtension) Shutdown(ctx context.Context) error { func (ml *memoryLimiterExtension) MustRefuse() bool { return ml.memLimiter.MustRefuse() } + +func (ml *memoryLimiterExtension) GetHTTPHandler(base http.Handler) (http.Handler, error) { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if ml.MustRefuse() { + http.Error(resp, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + base.ServeHTTP(resp, req) + }), nil +} + +func (ml *memoryLimiterExtension) GetGRPCServerOptions() ([]grpc.ServerOption, error) { + return []grpc.ServerOption{grpc.UnaryInterceptor( + func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + if ml.MustRefuse() { + return nil, status.Errorf(codes.ResourceExhausted, "RESOURCE_EXHAUSTED") + } + return handler(ctx, req) + }, + )}, nil +} diff --git a/extension/memorylimiterextension/memorylimiter_test.go b/extension/memorylimiterextension/memorylimiter_test.go index 6edbc3bd2d4..ad2970f9417 100644 --- a/extension/memorylimiterextension/memorylimiter_test.go +++ b/extension/memorylimiterextension/memorylimiter_test.go @@ -14,6 +14,7 @@ import ( "go.uber.org/zap" "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/extension/extensionmiddleware" "go.opentelemetry.io/collector/internal/memorylimiter" "go.opentelemetry.io/collector/internal/memorylimiter/iruntime" ) @@ -95,3 +96,8 @@ func TestMemoryPressureResponse(t *testing.T) { }) } } + +func TestMiddleware(t *testing.T) { + assert.Implements(t, (*extensionmiddleware.HTTPServer)(nil), &memoryLimiterExtension{}, "memoryLimiterExtension does not impplement extensionmiddleware.HTTPServer interface") + assert.Implements(t, (*extensionmiddleware.GRPCServer)(nil), &memoryLimiterExtension{}, "memoryLimiterExtension does not impplement extensionmiddleware.GRPCServer interface") +}