fix(otlp): prevent auth tokens from leaking in export error messages (#3360)

Author: bryantbiggs

opentelemetry-otlp/CHANGELOG.md

@@ -13,6 +13,12 @@
   silently sending unencrypted traffic. When a TLS feature is enabled and an `https://` endpoint is used without
   an explicit `.with_tls_config()`, a default `ClientTlsConfig` is automatically applied.
   [#3182](https://github.com/open-telemetry/opentelemetry-rust/issues/3182)
+- Prevent auth tokens from leaking in export error messages. gRPC and HTTP
+  exporter errors no longer include potentially sensitive server responses
+  (e.g., authentication tokens echoed back). Error messages returned to SDK
+  processors contain only the gRPC status code or HTTP status code. Full
+  details are logged at DEBUG level only.
+  [#3021](https://github.com/open-telemetry/opentelemetry-rust/issues/3021)
 - Add support for `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE` environment variable
   to configure metrics temporality. Accepted values: `cumulative` (default), `delta`,
   `lowmemory` (case-insensitive). Programmatic `.with_temporality()` overrides the env var.

opentelemetry-otlp/src/exporter/http/mod.rs

@@ -490,7 +490,17 @@ impl OtlpHttpClient {
 
         // Send request
         let response = client.send_bytes(request).await.map_err(|e| {
-            HttpExportError::new(0, format!("Network error: {e:?}")) // Network error
+            // Connection errors (e.g., "Connection refused", DNS failures) typically
+            // indicate user-side misconfigurations and don't contain sensitive data.
+            // We don't log at WARN here because SDK processors (BatchLogProcessor,
+            // BatchSpanProcessor, PeriodicReader) already log the returned error
+            // via otel_error!.
+            otel_debug!(
+                name: "HttpClient.NetworkError",
+                url = request_uri.as_str(),
+                error = format!("{e}")
+            );
+            HttpExportError::new(0, "HTTP export failed: network error".to_string())
         })?;
 
         let status_code = response.status().as_u16();
@@ -501,12 +511,18 @@ impl OtlpHttpClient {
             .map(|s| s.to_string());
 
         if !response.status().is_success() {
-            let message = format!(
-                "HTTP export failed. Url: {}, Status: {}, Response: {:?}",
-                request_uri,
-                status_code,
-                response.body()
+            // We don't log at WARN here because SDK processors (BatchLogProcessor,
+            // BatchSpanProcessor, PeriodicReader) already log the returned error
+            // via otel_error!. Response body may contain sensitive information
+            // (e.g., auth tokens echoed back by the server), so log it at DEBUG
+            // level only.
+            otel_debug!(
+                name: "HttpClient.StatusError",
+                status_code = status_code,
+                url = request_uri.as_str(),
+                response_body = format!("{:?}", response.body())
             );
+            let message = format!("HTTP export failed with status code: {status_code}");
             return Err(match retry_after {
                 Some(retry_after) => {
                     HttpExportError::with_retry_after(status_code, retry_after, message)

opentelemetry-otlp/src/exporter/tonic/logs.rs

@@ -94,8 +94,7 @@ impl LogExporter for TonicLogsClient {
                                 .interceptor
                                 .call(Request::new(()))
                                 .map_err(|e| {
-                                    // Convert interceptor errors to tonic::Status for retry classification
-                                    tonic::Status::internal(format!("interceptor error: {e:?}"))
+                                    super::handle_interceptor_error!("TonicLogsClient", e)
                                 })?
                                 .into_parts();
                             Ok((inner.client.clone(), m, e))
@@ -137,9 +136,9 @@ impl LogExporter for TonicLogsClient {
         .await
         {
             Ok(_) => Ok(()),
-            Err(tonic_status) => Err(OTelSdkError::InternalFailure(format!(
-                "export error: {tonic_status:?}"
-            ))),
+            Err(tonic_status) => {
+                super::handle_tonic_export_error!("TonicLogsClient", tonic_status)
+            }
         }
     }
 

opentelemetry-otlp/src/exporter/tonic/metrics.rs

@@ -85,9 +85,7 @@ impl MetricsClient for TonicMetricsClient {
                                 .interceptor
                                 .call(Request::new(()))
                                 .map_err(|e| {
-                                    tonic::Status::internal(format!(
-                                        "unexpected status while exporting {e:?}"
-                                    ))
+                                    super::handle_interceptor_error!("TonicMetricsClient", e)
                                 })?
                                 .into_parts();
                             Ok((inner.client.clone(), m, e))
@@ -127,9 +125,9 @@ impl MetricsClient for TonicMetricsClient {
         .await
         {
             Ok(_) => Ok(()),
-            Err(tonic_status) => Err(OTelSdkError::InternalFailure(format!(
-                "export error: {tonic_status:?}"
-            ))),
+            Err(tonic_status) => {
+                super::handle_tonic_export_error!("TonicMetricsClient", tonic_status)
+            }
         }
     }
 

opentelemetry-otlp/src/exporter/tonic/mod.rs

@@ -418,6 +418,72 @@ where
     operation().await
 }
 
+#[cfg(any(feature = "trace", feature = "metrics", feature = "logs"))]
+/// Log and convert a `tonic::Status` from a failed export into an `OTelSdkError`.
+///
+/// The gRPC code, message, and details are logged at DEBUG level only, since
+/// the message may contain sensitive information such as authentication tokens
+/// echoed back by the server.
+///
+/// The returned `OTelSdkError` never contains the gRPC message, only the code.
+/// We don't log at WARN here because SDK processors (BatchLogProcessor,
+/// BatchSpanProcessor, PeriodicReader) already log the returned error via
+/// `otel_error!`.
+///
+/// `$client_name` must be a string literal so that `concat!` can produce
+/// compile-time event names consistent with the codebase naming convention.
+macro_rules! handle_tonic_export_error {
+    ($client_name:literal, $tonic_status:expr) => {{
+        let status = &$tonic_status;
+        let code = status.code();
+        otel_debug!(
+            name: concat!($client_name, ".ExportFailed"),
+            grpc_code = format!("{:?}", code),
+            grpc_message = status.message(),
+            grpc_details = format!("{:?}", status.details())
+        );
+        Err(opentelemetry_sdk::error::OTelSdkError::InternalFailure(
+            format!(
+                concat!($client_name, " export failed with gRPC code: {:?}"),
+                code
+            ),
+        ))
+    }};
+}
+
+#[cfg(any(feature = "trace", feature = "metrics", feature = "logs"))]
+/// Log and convert a `tonic::Status` from a failed interceptor into a new
+/// `tonic::Status` suitable for retry classification.
+///
+/// Interceptor errors are always treated as potentially sensitive since
+/// interceptors are the primary mechanism for adding auth tokens. Only the
+/// gRPC code is included in the returned status message; the original message
+/// and details are logged at DEBUG level only.
+macro_rules! handle_interceptor_error {
+    ($client_name:literal, $e:expr) => {{
+        let status = &$e;
+        otel_debug!(
+            name: concat!($client_name, ".InterceptorFailed"),
+            grpc_code = format!("{:?}", status.code()),
+            grpc_message = status.message(),
+            grpc_details = format!("{:?}", status.details())
+        );
+        tonic::Status::internal(format!(
+            concat!(
+                $client_name,
+                " export failed in interceptor with gRPC code: {:?}"
+            ),
+            status.code()
+        ))
+    }};
+}
+
+// Make macros available to submodules (logs, trace, metrics).
+#[cfg(any(feature = "trace", feature = "metrics", feature = "logs"))]
+pub(crate) use handle_interceptor_error;
+#[cfg(any(feature = "trace", feature = "metrics", feature = "logs"))]
+pub(crate) use handle_tonic_export_error;
+
 fn merge_metadata_with_headers_from_env(
     metadata: MetadataMap,
     headers_from_env: HeaderMap,
@@ -1071,4 +1137,100 @@ mod tests {
             result.unwrap_err()
         );
     }
+
+    #[cfg(any(feature = "trace", feature = "metrics", feature = "logs"))]
+    mod error_handling_tests {
+        use opentelemetry::otel_debug;
+        use opentelemetry_sdk::error::OTelSdkError;
+
+        #[test]
+        fn export_error_includes_grpc_code_but_not_sensitive_message() {
+            let status = tonic::Status::unauthenticated("Bearer secret-token-123");
+            let result: Result<(), OTelSdkError> =
+                super::super::handle_tonic_export_error!("TestExporter", status);
+            let msg = format!("{}", result.unwrap_err());
+
+            assert!(
+                msg.contains("Unauthenticated"),
+                "Error should contain the gRPC code, got: {msg}"
+            );
+            assert!(
+                !msg.contains("secret-token-123"),
+                "Error must not contain the sensitive token, got: {msg}"
+            );
+        }
+
+        #[test]
+        fn export_error_includes_exporter_name() {
+            let status = tonic::Status::unavailable("connection refused");
+            let result: Result<(), OTelSdkError> =
+                super::super::handle_tonic_export_error!("TonicLogsClient", status);
+            let msg = format!("{}", result.unwrap_err());
+
+            assert!(
+                msg.contains("TonicLogsClient"),
+                "Error should identify the exporter, got: {msg}"
+            );
+        }
+
+        #[test]
+        fn export_error_never_includes_grpc_message() {
+            // Neither connection nor sensitive codes should leak the message
+            // into the returned error (messages are only logged, not returned)
+            let statuses = [
+                tonic::Status::unavailable("safe connection info"),
+                tonic::Status::unknown("safe connection info"),
+                tonic::Status::deadline_exceeded("safe connection info"),
+                tonic::Status::resource_exhausted("safe connection info"),
+                tonic::Status::aborted("safe connection info"),
+                tonic::Status::cancelled("safe connection info"),
+                tonic::Status::unauthenticated("Bearer my-secret-token"),
+                tonic::Status::permission_denied("Bearer my-secret-token"),
+                tonic::Status::internal("Bearer my-secret-token"),
+            ];
+            for status in &statuses {
+                let result: Result<(), OTelSdkError> =
+                    super::super::handle_tonic_export_error!("TestExporter", status);
+                let msg = format!("{}", result.unwrap_err());
+                assert!(
+                    msg.contains("TestExporter export failed with gRPC code"),
+                    "Expected structured error message, got: {msg}"
+                );
+                assert!(
+                    !msg.contains("safe connection info") && !msg.contains("my-secret-token"),
+                    "Error message should not include the gRPC message, got: {msg}"
+                );
+            }
+        }
+
+        #[test]
+        fn interceptor_error_returns_internal_status_without_sensitive_data() {
+            let original = tonic::Status::unauthenticated("Bearer secret");
+            let result = super::super::handle_interceptor_error!("TestExporter", original);
+
+            assert_eq!(result.code(), tonic::Code::Internal);
+            assert!(
+                result.message().contains("Unauthenticated"),
+                "Interceptor error should contain original gRPC code, got: {}",
+                result.message()
+            );
+            assert!(
+                !result.message().contains("secret"),
+                "Interceptor error must not leak sensitive data, got: {}",
+                result.message()
+            );
+        }
+
+        #[test]
+        fn interceptor_error_includes_exporter_name() {
+            let original = tonic::Status::internal("some error");
+            let result = super::super::handle_interceptor_error!("TonicTracesClient", original);
+
+            assert!(
+                result.message().contains("TonicTracesClient"),
+                "Interceptor error should identify the exporter, got: {}",
+                result.message()
+            );
+        }
+    }
 }

opentelemetry-otlp/src/exporter/tonic/trace.rs

@@ -96,8 +96,7 @@ impl SpanExporter for TonicTracesClient {
                                 .interceptor
                                 .call(Request::new(()))
                                 .map_err(|e| {
-                                    // Convert interceptor errors to tonic::Status for retry classification
-                                    tonic::Status::internal(format!("interceptor error: {e:?}"))
+                                    super::handle_interceptor_error!("TonicTracesClient", e)
                                 })?
                                 .into_parts();
                             Ok((inner.client.clone(), m, e))
@@ -140,9 +139,9 @@ impl SpanExporter for TonicTracesClient {
         .await
         {
             Ok(_) => Ok(()),
-            Err(tonic_status) => Err(OTelSdkError::InternalFailure(format!(
-                "export error: {tonic_status:?}"
-            ))),
+            Err(tonic_status) => {
+                super::handle_tonic_export_error!("TonicTracesClient", tonic_status)
+            }
         }
     }
 

opentelemetry-otlp/tests/integration_test/tests/metrics_roundtrip.rs

@@ -21,6 +21,7 @@ mod metrictests_roundtrip {
     use integration_test_runner::metric_helpers::{
         self, validate_metrics_against_results, SLEEP_DURATION,
     };
+    use opentelemetry::metrics::MeterProvider;
 
     use super::*;
 
@@ -54,7 +55,7 @@ mod metrictests_roundtrip {
         const METER_NAME: &str = "test_u64_counter_meter";
 
         // Add data to u64_counter
-        let meter = opentelemetry::global::meter_provider().meter(METER_NAME);
+        let meter = meter_provider.meter(METER_NAME);
 
         let counter = meter.u64_counter("counter_u64").build();
         counter.add(
@@ -80,7 +81,7 @@ mod metrictests_roundtrip {
         const METER_NAME: &str = "test_histogram_meter";
 
         // Add data to histogram
-        let meter = opentelemetry::global::meter_provider().meter(METER_NAME);
+        let meter = meter_provider.meter(METER_NAME);
         let histogram = meter.u64_histogram("example_histogram").build();
         histogram.record(42, &[KeyValue::new("mykey3", "myvalue4")]);
 
@@ -98,7 +99,7 @@ mod metrictests_roundtrip {
         const METER_NAME: &str = "test_up_down_meter";
 
         // Add data to up_down_counter
-        let meter = opentelemetry::global::meter_provider().meter(METER_NAME);
+        let meter = meter_provider.meter(METER_NAME);
         let up_down_counter = meter.i64_up_down_counter("example_up_down_counter").build();
         up_down_counter.add(-1, &[KeyValue::new("mykey5", "myvalue5")]);