From d2ed276a9215053790ddead2d6479a6beacd28a7 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Fri, 16 Aug 2024 19:46:40 -0700 Subject: [PATCH] Enable batch sender in oltpexporter (#10846) #### Description This PR adds opt-in support to oltp exporter for the experimental batch sender (https://github.com/open-telemetry/opentelemetry-collector/issues/8122). By default batch sender is not enabled. Similar: * https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/34238 * https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/32563 #### Link to tracking issue Resolves https://github.com/open-telemetry/opentelemetry-collector/issues/10834 #### Testing `exporter/otlpexporter/config_test.go` #### Documentation Updated the `oltpexporter` README to point to `exporterhelper` README for batching. --------- Co-authored-by: Dmitrii Anoshin --- .../add-batching-option-to-otlp-exporter.yaml | 25 +++++++++++++++++++ exporter/otlpexporter/README.md | 2 +- exporter/otlpexporter/config.go | 5 ++++ exporter/otlpexporter/config_test.go | 11 ++++++++ exporter/otlpexporter/factory.go | 11 +++++++- exporter/otlpexporter/testdata/config.yaml | 5 ++++ 6 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 .chloggen/add-batching-option-to-otlp-exporter.yaml diff --git a/.chloggen/add-batching-option-to-otlp-exporter.yaml b/.chloggen/add-batching-option-to-otlp-exporter.yaml new file mode 100644 index 00000000000..b83465823df --- /dev/null +++ b/.chloggen/add-batching-option-to-otlp-exporter.yaml @@ -0,0 +1,25 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: exporter/otlp + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Add batching option to otlp exporter" + +# One or more tracking issues or pull requests related to the change +issues: [8122] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/otlpexporter/README.md b/exporter/otlpexporter/README.md index b4a5c7b055d..13589d6aac9 100644 --- a/exporter/otlpexporter/README.md +++ b/exporter/otlpexporter/README.md @@ -59,4 +59,4 @@ Several helper files are leveraged to provide additional capabilities automatica - [gRPC settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configgrpc/README.md) - [TLS and mTLS settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md) -- [Queuing, retry and timeout settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) +- [Queuing, batching, retry and timeout settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) diff --git a/exporter/otlpexporter/config.go b/exporter/otlpexporter/config.go index f1d5b668e54..2aec2995054 100644 --- a/exporter/otlpexporter/config.go +++ b/exporter/otlpexporter/config.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/configgrpc" "go.opentelemetry.io/collector/config/configretry" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" ) @@ -23,6 +24,10 @@ type Config struct { QueueConfig exporterhelper.QueueSettings `mapstructure:"sending_queue"` RetryConfig configretry.BackOffConfig `mapstructure:"retry_on_failure"` + // Experimental: This configuration is at the early stage of development and may change without backward compatibility + // until https://github.com/open-telemetry/opentelemetry-collector/issues/8122 is resolved + BatcherConfig exporterbatcher.Config `mapstructure:"batcher"` + configgrpc.ClientConfig `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. } diff --git a/exporter/otlpexporter/config_test.go b/exporter/otlpexporter/config_test.go index a29d0182860..7a855d47b51 100644 --- a/exporter/otlpexporter/config_test.go +++ b/exporter/otlpexporter/config_test.go @@ -19,6 +19,7 @@ import ( "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" ) @@ -53,6 +54,16 @@ func TestUnmarshalConfig(t *testing.T) { NumConsumers: 2, QueueSize: 10, }, + BatcherConfig: exporterbatcher.Config{ + Enabled: true, + FlushTimeout: 200 * time.Millisecond, + MinSizeConfig: exporterbatcher.MinSizeConfig{ + MinSizeItems: 1000, + }, + MaxSizeConfig: exporterbatcher.MaxSizeConfig{ + MaxSizeItems: 10000, + }, + }, ClientConfig: configgrpc.ClientConfig{ Headers: map[string]configopaque.String{ "can you have a . here?": "F0000000-0000-0000-0000-000000000000", diff --git a/exporter/otlpexporter/factory.go b/exporter/otlpexporter/factory.go index c06a6c605b2..84952defdf3 100644 --- a/exporter/otlpexporter/factory.go +++ b/exporter/otlpexporter/factory.go @@ -13,6 +13,7 @@ import ( "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" "go.opentelemetry.io/collector/exporter/otlpexporter/internal/metadata" ) @@ -29,10 +30,14 @@ func NewFactory() exporter.Factory { } func createDefaultConfig() component.Config { + batcherCfg := exporterbatcher.NewDefaultConfig() + batcherCfg.Enabled = false + return &Config{ TimeoutSettings: exporterhelper.NewDefaultTimeoutSettings(), RetryConfig: configretry.NewDefaultBackOffConfig(), QueueConfig: exporterhelper.NewDefaultQueueSettings(), + BatcherConfig: batcherCfg, ClientConfig: configgrpc.ClientConfig{ Headers: map[string]configopaque.String{}, // Default to gzip compression @@ -56,8 +61,10 @@ func createTracesExporter( exporterhelper.WithTimeout(oCfg.TimeoutSettings), exporterhelper.WithRetry(oCfg.RetryConfig), exporterhelper.WithQueue(oCfg.QueueConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), exporterhelper.WithStart(oce.start), - exporterhelper.WithShutdown(oce.shutdown)) + exporterhelper.WithShutdown(oce.shutdown), + ) } func createMetricsExporter( @@ -73,6 +80,7 @@ func createMetricsExporter( exporterhelper.WithTimeout(oCfg.TimeoutSettings), exporterhelper.WithRetry(oCfg.RetryConfig), exporterhelper.WithQueue(oCfg.QueueConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), ) @@ -91,6 +99,7 @@ func createLogsExporter( exporterhelper.WithTimeout(oCfg.TimeoutSettings), exporterhelper.WithRetry(oCfg.RetryConfig), exporterhelper.WithQueue(oCfg.QueueConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), ) diff --git a/exporter/otlpexporter/testdata/config.yaml b/exporter/otlpexporter/testdata/config.yaml index 7716736b678..d26631053e1 100644 --- a/exporter/otlpexporter/testdata/config.yaml +++ b/exporter/otlpexporter/testdata/config.yaml @@ -14,6 +14,11 @@ retry_on_failure: multiplier: 1.3 max_interval: 60s max_elapsed_time: 10m +batcher: + enabled: true + flush_timeout: 200ms + min_size_items: 1000 + max_size_items: 10000 auth: authenticator: nop headers: