Add retry dropped item metrics and an exhausted retry error marker for exporter helper retries

Signed-off-by: Israel Blancas <[email protected]>

Author: iblancasa

.chloggen/13956.yaml

@@ -0,0 +1,26 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: pkg/exporterhelper
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add retry dropped item metrics and an exhausted retry error marker for exporter helper retries.
+
+# One or more tracking issues or pull requests related to the change
+issues: [13956]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |-
+  New counters `otelcol_exporter_retry_dropped_{spans,metric_points,log_records}` capture items discarded after exhausting retries while `IsRetriesExhaustedErr` detects the terminal retry outcome.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/workflows/build-and-test.yml

@@ -93,7 +93,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
         with:
-          go-version: stable
+          go-version: oldstable
           cache: false
       - name: Cache Go
         id: go-cache

cmd/otelcorecol/go.mod

@@ -4,6 +4,8 @@ module go.opentelemetry.io/collector/cmd/otelcorecol
 
 go 1.24.0
 
+toolchain go1.24.1
+
 require (
 	go.opentelemetry.io/collector/component v1.45.0
 	go.opentelemetry.io/collector/confmap v1.45.0

exporter/exporterhelper/documentation.md

@@ -62,6 +62,30 @@ Current size of the retry queue (in batches). [Alpha]
 | ---- | ----------- | ---------- | --------- |
 | {batches} | Gauge | Int | Alpha |
 
+### otelcol_exporter_retry_dropped_log_records
+
+Number of log records dropped after exhausting configured retries. [Alpha]
+
+| Unit | Metric Type | Value Type | Monotonic | Stability |
+| ---- | ----------- | ---------- | --------- | --------- |
+| {records} | Sum | Int | true | Alpha |
+
+### otelcol_exporter_retry_dropped_metric_points
+
+Number of metric points dropped after exhausting configured retries. [Alpha]
+
+| Unit | Metric Type | Value Type | Monotonic | Stability |
+| ---- | ----------- | ---------- | --------- | --------- |
+| {datapoints} | Sum | Int | true | Alpha |
+
+### otelcol_exporter_retry_dropped_spans
+
+Number of spans dropped after exhausting configured retries. [Alpha]
+
+| Unit | Metric Type | Value Type | Monotonic | Stability |
+| ---- | ----------- | ---------- | --------- | --------- |
+| {spans} | Sum | Int | true | Alpha |
+
 ### otelcol_exporter_send_failed_log_records
 
 Number of log records in failed attempts to send to destination. [Alpha]

exporter/exporterhelper/internal/experr/err.go

@@ -3,9 +3,7 @@
 
 package experr // import "go.opentelemetry.io/collector/exporter/exporterhelper/internal/experr"
 
-import (
-	"errors"
-)
+import "errors"
 
 type shutdownErr struct {
 	err error
@@ -27,3 +25,24 @@ func IsShutdownErr(err error) bool {
 	var sdErr shutdownErr
 	return errors.As(err, &sdErr)
 }
+
+type retriesExhaustedErr struct {
+	err error
+}
+
+func NewRetriesExhaustedErr(err error) error {
+	return retriesExhaustedErr{err: err}
+}
+
+func (r retriesExhaustedErr) Error() string {
+	return "retries exhausted: " + r.err.Error()
+}
+
+func (r retriesExhaustedErr) Unwrap() error {
+	return r.err
+}
+
+func IsRetriesExhaustedErr(err error) bool {
+	var reErr retriesExhaustedErr
+	return errors.As(err, &reErr)
+}

exporter/exporterhelper/internal/experr/err_test.go

@@ -22,3 +22,15 @@ func TestIsShutdownErr(t *testing.T) {
 	err = NewShutdownErr(err)
 	require.True(t, IsShutdownErr(err))
 }
+
+func TestNewRetriesExhaustedErr(t *testing.T) {
+	err := NewRetriesExhaustedErr(errors.New("another error"))
+	assert.Equal(t, "retries exhausted: another error", err.Error())
+}
+
+func TestIsRetriesExhaustedErr(t *testing.T) {
+	err := errors.New("testError")
+	require.False(t, IsRetriesExhaustedErr(err))
+	err = NewRetriesExhaustedErr(err)
+	require.True(t, IsRetriesExhaustedErr(err))
+}

exporter/exporterhelper/internal/metadata/generated_telemetry.go

@@ -13,6 +13,7 @@ import (
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/exporter"
+	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/experr"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/metadata"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/queuebatch"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/request"
@@ -40,13 +41,14 @@ type obsReportSender[K request.Request] struct {
 	component.StartFunc
 	component.ShutdownFunc
 
-	spanName        string
-	tracer          trace.Tracer
-	spanAttrs       trace.SpanStartEventOption
-	metricAttr      metric.MeasurementOption
-	itemsSentInst   metric.Int64Counter
-	itemsFailedInst metric.Int64Counter
-	next            sender.Sender[K]
+	spanName              string
+	tracer                trace.Tracer
+	spanAttrs             trace.SpanStartEventOption
+	metricAttr            metric.MeasurementOption
+	itemsSentInst         metric.Int64Counter
+	itemsFailedInst       metric.Int64Counter
+	itemsRetryDroppedInst metric.Int64Counter
+	next                  sender.Sender[K]
 }
 
 func newObsReportSender[K request.Request](set exporter.Settings, signal pipeline.Signal, next sender.Sender[K]) (sender.Sender[K], error) {
@@ -70,14 +72,17 @@ func newObsReportSender[K request.Request](set exporter.Settings, signal pipelin
 	case pipeline.SignalTraces:
 		or.itemsSentInst = telemetryBuilder.ExporterSentSpans
 		or.itemsFailedInst = telemetryBuilder.ExporterSendFailedSpans
+		or.itemsRetryDroppedInst = telemetryBuilder.ExporterRetryDroppedSpans
 
 	case pipeline.SignalMetrics:
 		or.itemsSentInst = telemetryBuilder.ExporterSentMetricPoints
 		or.itemsFailedInst = telemetryBuilder.ExporterSendFailedMetricPoints
+		or.itemsRetryDroppedInst = telemetryBuilder.ExporterRetryDroppedMetricPoints
 
 	case pipeline.SignalLogs:
 		or.itemsSentInst = telemetryBuilder.ExporterSentLogRecords
 		or.itemsFailedInst = telemetryBuilder.ExporterSendFailedLogRecords
+		or.itemsRetryDroppedInst = telemetryBuilder.ExporterRetryDroppedLogRecords
 	}
 
 	return or, nil
@@ -116,6 +121,10 @@ func (ors *obsReportSender[K]) endOp(ctx context.Context, numLogRecords int, err
 	if ors.itemsFailedInst != nil {
 		ors.itemsFailedInst.Add(ctx, numFailedToSend, ors.metricAttr)
 	}
+	// Count drops after retries were exhausted.
+	if err != nil && ors.itemsRetryDroppedInst != nil && experr.IsRetriesExhaustedErr(err) {
+		ors.itemsRetryDroppedInst.Add(ctx, numFailedToSend, ors.metricAttr)
+	}
 
 	span := trace.SpanFromContext(ctx)
 	defer span.End()