From 98ec533132d00bc038c400ecd5ac01ae860a2499 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Thu, 8 Aug 2024 20:52:16 -0700 Subject: [PATCH 01/13] Enable batch sender in oltpexporter --- exporter/otlpexporter/README.md | 2 +- exporter/otlpexporter/config.go | 4 ++++ exporter/otlpexporter/config_test.go | 11 +++++++++++ exporter/otlpexporter/factory.go | 11 ++++++++++- exporter/otlpexporter/testdata/config.yaml | 5 +++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/exporter/otlpexporter/README.md b/exporter/otlpexporter/README.md index b4a5c7b055d..13589d6aac9 100644 --- a/exporter/otlpexporter/README.md +++ b/exporter/otlpexporter/README.md @@ -59,4 +59,4 @@ Several helper files are leveraged to provide additional capabilities automatica - [gRPC settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configgrpc/README.md) - [TLS and mTLS settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md) -- [Queuing, retry and timeout settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) +- [Queuing, batching, retry and timeout settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) diff --git a/exporter/otlpexporter/config.go b/exporter/otlpexporter/config.go index f1d5b668e54..9f499f1e6b1 100644 --- a/exporter/otlpexporter/config.go +++ b/exporter/otlpexporter/config.go @@ -23,6 +23,10 @@ type Config struct { QueueConfig exporterhelper.QueueSettings `mapstructure:"sending_queue"` RetryConfig configretry.BackOffConfig `mapstructure:"retry_on_failure"` + // Experimental: This configuration is at the early stage of development and may change without backward compatibility + // until https://github.com/open-telemetry/opentelemetry-collector/issues/8122 is resolved. + BatcherConfig exporterbatcher.Config `mapstructure:"batcher"` + configgrpc.ClientConfig `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. } diff --git a/exporter/otlpexporter/config_test.go b/exporter/otlpexporter/config_test.go index a29d0182860..e0d2d57fc67 100644 --- a/exporter/otlpexporter/config_test.go +++ b/exporter/otlpexporter/config_test.go @@ -19,6 +19,7 @@ import ( "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" ) @@ -53,6 +54,16 @@ func TestUnmarshalConfig(t *testing.T) { NumConsumers: 2, QueueSize: 10, }, + BatcherConfig: exporterbatcher.Config{ + Enabled: true, + FlushTimeout: time.Second, + MinSizeConfig: exporterbatcher.MinSizeConfig{ + MinSizeItems: 1, + }, + MaxSizeConfig: exporterbatcher.MaxSizeConfig{ + MaxSizeItems: 10, + }, + }, ClientConfig: configgrpc.ClientConfig{ Headers: map[string]configopaque.String{ "can you have a . here?": "F0000000-0000-0000-0000-000000000000", diff --git a/exporter/otlpexporter/factory.go b/exporter/otlpexporter/factory.go index c06a6c605b2..2c5aef3a965 100644 --- a/exporter/otlpexporter/factory.go +++ b/exporter/otlpexporter/factory.go @@ -13,6 +13,7 @@ import ( "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" "go.opentelemetry.io/collector/exporter/otlpexporter/internal/metadata" ) @@ -29,10 +30,14 @@ func NewFactory() exporter.Factory { } func createDefaultConfig() component.Config { + batcherCfg := exporterbatcher.NewDefaultConfig() + batcherCfg.Enabled = false + return &Config{ TimeoutSettings: exporterhelper.NewDefaultTimeoutSettings(), RetryConfig: configretry.NewDefaultBackOffConfig(), QueueConfig: exporterhelper.NewDefaultQueueSettings(), + BatcherConfig: batcherCfg, ClientConfig: configgrpc.ClientConfig{ Headers: map[string]configopaque.String{}, // Default to gzip compression @@ -57,7 +62,9 @@ func createTracesExporter( exporterhelper.WithRetry(oCfg.RetryConfig), exporterhelper.WithQueue(oCfg.QueueConfig), exporterhelper.WithStart(oce.start), - exporterhelper.WithShutdown(oce.shutdown)) + exporterhelper.WithShutdown(oce.shutdown), + exporterhelper.WithBatcher(cfg.BatcherConfig), + ) } func createMetricsExporter( @@ -75,6 +82,7 @@ func createMetricsExporter( exporterhelper.WithQueue(oCfg.QueueConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), + exporterhelper.WithBatcher(cfg.BatcherConfig), ) } @@ -93,5 +101,6 @@ func createLogsExporter( exporterhelper.WithQueue(oCfg.QueueConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), + exporterhelper.WithBatcher(cfg.BatcherConfig), ) } diff --git a/exporter/otlpexporter/testdata/config.yaml b/exporter/otlpexporter/testdata/config.yaml index 7716736b678..80740a3a647 100644 --- a/exporter/otlpexporter/testdata/config.yaml +++ b/exporter/otlpexporter/testdata/config.yaml @@ -14,6 +14,11 @@ retry_on_failure: multiplier: 1.3 max_interval: 60s max_elapsed_time: 10m +batcher: + enabled: true + flush_timeout: 1s + min_size_items: 1 + max_size_items: 10 auth: authenticator: nop headers: From 55ec1fcc85e9a2a0b11cd61acfc44217d6bf8799 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Mon, 12 Aug 2024 15:04:57 -0700 Subject: [PATCH 02/13] Update changelog --- .../add-batching-option-to-oltp-exporter.yaml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .chloggen/add-batching-option-to-oltp-exporter.yaml diff --git a/.chloggen/add-batching-option-to-oltp-exporter.yaml b/.chloggen/add-batching-option-to-oltp-exporter.yaml new file mode 100644 index 00000000000..b609ecc8893 --- /dev/null +++ b/.chloggen/add-batching-option-to-oltp-exporter.yaml @@ -0,0 +1,25 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: otlp + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Add batching option to otlp receiver" + +# One or more tracking issues or pull requests related to the change +issues: [8122] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] \ No newline at end of file From 346e8dfe3e2b17d8ddaf7249129727e12a0e3d01 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Mon, 12 Aug 2024 20:11:01 -0700 Subject: [PATCH 03/13] Fix merge gate failure --- .chloggen/add-batching-option-to-oltp-exporter.yaml | 2 +- exporter/otlpexporter/config.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.chloggen/add-batching-option-to-oltp-exporter.yaml b/.chloggen/add-batching-option-to-oltp-exporter.yaml index b609ecc8893..c58e2f24d89 100644 --- a/.chloggen/add-batching-option-to-oltp-exporter.yaml +++ b/.chloggen/add-batching-option-to-oltp-exporter.yaml @@ -4,7 +4,7 @@ change_type: enhancement # The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) -component: otlp +component: otlpreceiver # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). note: "Add batching option to otlp receiver" diff --git a/exporter/otlpexporter/config.go b/exporter/otlpexporter/config.go index 9f499f1e6b1..a69300c290a 100644 --- a/exporter/otlpexporter/config.go +++ b/exporter/otlpexporter/config.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/configgrpc" "go.opentelemetry.io/collector/config/configretry" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" ) From f708d8111f931fc8eac5194eddfe4a49dabcbeca Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Tue, 13 Aug 2024 14:35:54 -0700 Subject: [PATCH 04/13] Update change log --- ...xporter.yaml => add-batching-option-to-otlp-exporter.yaml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .chloggen/{add-batching-option-to-oltp-exporter.yaml => add-batching-option-to-otlp-exporter.yaml} (93%) diff --git a/.chloggen/add-batching-option-to-oltp-exporter.yaml b/.chloggen/add-batching-option-to-otlp-exporter.yaml similarity index 93% rename from .chloggen/add-batching-option-to-oltp-exporter.yaml rename to .chloggen/add-batching-option-to-otlp-exporter.yaml index c58e2f24d89..e177472443f 100644 --- a/.chloggen/add-batching-option-to-oltp-exporter.yaml +++ b/.chloggen/add-batching-option-to-otlp-exporter.yaml @@ -4,10 +4,10 @@ change_type: enhancement # The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) -component: otlpreceiver +component: exporter/otlp # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: "Add batching option to otlp receiver" +note: "Add batching option to otlp exporter" # One or more tracking issues or pull requests related to the change issues: [8122] From fc7ec1e079cf457e32a4ac23b7ad29f1bd03f341 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Tue, 13 Aug 2024 23:24:33 -0700 Subject: [PATCH 05/13] Empty commit From 959cb247585bed3e77abc0f63f91e553a441f4b5 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Wed, 14 Aug 2024 11:28:32 -0700 Subject: [PATCH 06/13] Make linter happy --- exporter/otlpexporter/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/otlpexporter/config.go b/exporter/otlpexporter/config.go index a69300c290a..6466a54c866 100644 --- a/exporter/otlpexporter/config.go +++ b/exporter/otlpexporter/config.go @@ -26,7 +26,7 @@ type Config struct { // Experimental: This configuration is at the early stage of development and may change without backward compatibility // until https://github.com/open-telemetry/opentelemetry-collector/issues/8122 is resolved. - BatcherConfig exporterbatcher.Config `mapstructure:"batcher"` + BatcherConfig exporterbatcher.Config `mapstructure:"batcher"` configgrpc.ClientConfig `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. } From 3db57af719ac5a38dc196be746e70dbc212687c3 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Wed, 14 Aug 2024 11:29:26 -0700 Subject: [PATCH 07/13] Empty commit. From 52db7edab3780189484883c406b2b9489a6018f5 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Wed, 14 Aug 2024 11:41:42 -0700 Subject: [PATCH 08/13] Trigger precommit --- exporter/otlpexporter/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/otlpexporter/config.go b/exporter/otlpexporter/config.go index 6466a54c866..2aec2995054 100644 --- a/exporter/otlpexporter/config.go +++ b/exporter/otlpexporter/config.go @@ -25,7 +25,7 @@ type Config struct { RetryConfig configretry.BackOffConfig `mapstructure:"retry_on_failure"` // Experimental: This configuration is at the early stage of development and may change without backward compatibility - // until https://github.com/open-telemetry/opentelemetry-collector/issues/8122 is resolved. + // until https://github.com/open-telemetry/opentelemetry-collector/issues/8122 is resolved BatcherConfig exporterbatcher.Config `mapstructure:"batcher"` configgrpc.ClientConfig `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. From 3fdba2cb0bdd51b5c1e2dc3a601d03bd83d406a7 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Wed, 14 Aug 2024 13:26:49 -0700 Subject: [PATCH 09/13] Fix build failure --- exporter/otlpexporter/factory.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/otlpexporter/factory.go b/exporter/otlpexporter/factory.go index 2c5aef3a965..a653648737e 100644 --- a/exporter/otlpexporter/factory.go +++ b/exporter/otlpexporter/factory.go @@ -63,7 +63,7 @@ func createTracesExporter( exporterhelper.WithQueue(oCfg.QueueConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), - exporterhelper.WithBatcher(cfg.BatcherConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), ) } @@ -82,7 +82,7 @@ func createMetricsExporter( exporterhelper.WithQueue(oCfg.QueueConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), - exporterhelper.WithBatcher(cfg.BatcherConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), ) } @@ -101,6 +101,6 @@ func createLogsExporter( exporterhelper.WithQueue(oCfg.QueueConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), - exporterhelper.WithBatcher(cfg.BatcherConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), ) } From 2783dec6f3adeb74f44b8f6973eab3f7bb134626 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Thu, 15 Aug 2024 17:21:43 -0700 Subject: [PATCH 10/13] Update exporter/otlpexporter/testdata/config.yaml Co-authored-by: Dmitrii Anoshin --- exporter/otlpexporter/testdata/config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/otlpexporter/testdata/config.yaml b/exporter/otlpexporter/testdata/config.yaml index 80740a3a647..d22675de488 100644 --- a/exporter/otlpexporter/testdata/config.yaml +++ b/exporter/otlpexporter/testdata/config.yaml @@ -16,9 +16,9 @@ retry_on_failure: max_elapsed_time: 10m batcher: enabled: true - flush_timeout: 1s - min_size_items: 1 - max_size_items: 10 + flush_timeout: 200ms + min_size_items: 1000 + max_size_items: 10000 auth: authenticator: nop headers: From 8cabd9140ad5a8e1741d0c16f1310b07cdb058fb Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Thu, 15 Aug 2024 17:59:38 -0700 Subject: [PATCH 11/13] Moving WithBatcher to behind WithQueue --- exporter/otlpexporter/factory.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/otlpexporter/factory.go b/exporter/otlpexporter/factory.go index a653648737e..84952defdf3 100644 --- a/exporter/otlpexporter/factory.go +++ b/exporter/otlpexporter/factory.go @@ -61,9 +61,9 @@ func createTracesExporter( exporterhelper.WithTimeout(oCfg.TimeoutSettings), exporterhelper.WithRetry(oCfg.RetryConfig), exporterhelper.WithQueue(oCfg.QueueConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), - exporterhelper.WithBatcher(oCfg.BatcherConfig), ) } @@ -80,9 +80,9 @@ func createMetricsExporter( exporterhelper.WithTimeout(oCfg.TimeoutSettings), exporterhelper.WithRetry(oCfg.RetryConfig), exporterhelper.WithQueue(oCfg.QueueConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), - exporterhelper.WithBatcher(oCfg.BatcherConfig), ) } @@ -99,8 +99,8 @@ func createLogsExporter( exporterhelper.WithTimeout(oCfg.TimeoutSettings), exporterhelper.WithRetry(oCfg.RetryConfig), exporterhelper.WithQueue(oCfg.QueueConfig), + exporterhelper.WithBatcher(oCfg.BatcherConfig), exporterhelper.WithStart(oce.start), exporterhelper.WithShutdown(oce.shutdown), - exporterhelper.WithBatcher(oCfg.BatcherConfig), ) } From 52d7e81836c393a7c78e1a5e313812be42543c93 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Fri, 16 Aug 2024 13:17:51 -0700 Subject: [PATCH 12/13] Fix config test --- exporter/otlpexporter/config_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/otlpexporter/config_test.go b/exporter/otlpexporter/config_test.go index e0d2d57fc67..7a855d47b51 100644 --- a/exporter/otlpexporter/config_test.go +++ b/exporter/otlpexporter/config_test.go @@ -56,12 +56,12 @@ func TestUnmarshalConfig(t *testing.T) { }, BatcherConfig: exporterbatcher.Config{ Enabled: true, - FlushTimeout: time.Second, + FlushTimeout: 200 * time.Millisecond, MinSizeConfig: exporterbatcher.MinSizeConfig{ - MinSizeItems: 1, + MinSizeItems: 1000, }, MaxSizeConfig: exporterbatcher.MaxSizeConfig{ - MaxSizeItems: 10, + MaxSizeItems: 10000, }, }, ClientConfig: configgrpc.ClientConfig{ From 50cb789159f8d513aa362a4ad021172aa390b162 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Fri, 16 Aug 2024 13:23:34 -0700 Subject: [PATCH 13/13] Fix some format issue --- .chloggen/add-batching-option-to-otlp-exporter.yaml | 2 +- exporter/otlpexporter/testdata/config.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.chloggen/add-batching-option-to-otlp-exporter.yaml b/.chloggen/add-batching-option-to-otlp-exporter.yaml index e177472443f..b83465823df 100644 --- a/.chloggen/add-batching-option-to-otlp-exporter.yaml +++ b/.chloggen/add-batching-option-to-otlp-exporter.yaml @@ -22,4 +22,4 @@ subtext: # Include 'user' if the change is relevant to end users. # Include 'api' if there is a change to a library API. # Default: '[user]' -change_logs: [user] \ No newline at end of file +change_logs: [user] diff --git a/exporter/otlpexporter/testdata/config.yaml b/exporter/otlpexporter/testdata/config.yaml index d22675de488..d26631053e1 100644 --- a/exporter/otlpexporter/testdata/config.yaml +++ b/exporter/otlpexporter/testdata/config.yaml @@ -15,10 +15,10 @@ retry_on_failure: max_interval: 60s max_elapsed_time: 10m batcher: - enabled: true - flush_timeout: 200ms - min_size_items: 1000 - max_size_items: 10000 + enabled: true + flush_timeout: 200ms + min_size_items: 1000 + max_size_items: 10000 auth: authenticator: nop headers: