POTEL 63 - Let OpenTelemetry handle extracting and injecting tracing information (#3953)

* bump OTel to 2.10.0

* support DB_QUERY_TEXT

* changelog

* change bom version for otel

* Replace OTel ContextStorage wrapper with ContextStorageProvider

* bump spring boot 3.4

* fix twp

* Use null sampled for TwP

* make todo url configurable

* Build PropagationContext from sampling decision in OTel instead of incoming headers

* Format code

* changelog

* Revert "make todo url configurable"

This reverts commit a90f316.

* disable continueTrace and injecting tracing headers for ignored span origins

* also change apollo and okhttp

* changelog

---------

Co-authored-by: Sentry Github Bot <[email protected]>

Author: adinauer

CHANGELOG.md

@@ -10,6 +10,8 @@
   - In `TransactionContext.fromPropagationContext` when there is no parent sampling decision, keep the decision `null` so a new sampling decision is made instead of defaulting to `false`
 - Defer sampling decision by setting `sampled` to `null` in `PropagationContext` when using OpenTelemetry in case of an incoming defer sampling `sentry-trace` header. ([#3945](https://github.com/getsentry/sentry-java/pull/3945))
 - Build `PropagationContext` from `SamplingDecision` made by `SentrySampler` instead of parsing headers and potentially ignoring a sampling decision in case a `sentry-trace` header comes in with deferred sampling decision. ([#3947](https://github.com/getsentry/sentry-java/pull/3947))
+- Let OpenTelemetry handle extracting and injecting tracing information ([#3953](https://github.com/getsentry/sentry-java/pull/3953))
+  - Our integrations no longer call `.continueTrace` and also do not inject tracing headers if the integration has been added to `ignoredSpanOrigins`
 
 ## 8.0.0-rc.1
 

sentry-apollo-3/src/main/java/io/sentry/apollo3/SentryApollo3HttpInterceptor.kt

@@ -31,6 +31,7 @@ import io.sentry.util.HttpUtils
 import io.sentry.util.IntegrationUtils.addIntegrationToSdkVersion
 import io.sentry.util.Platform
 import io.sentry.util.PropagationTargetsUtils
+import io.sentry.util.SpanUtils
 import io.sentry.util.TracingUtils
 import io.sentry.util.UrlUtils
 import io.sentry.vendor.Base64
@@ -120,11 +121,13 @@ class SentryApollo3HttpInterceptor @JvmOverloads constructor(
     private fun maybeAddTracingHeaders(scopes: IScopes, request: HttpRequest, span: ISpan?): HttpRequest {
         var cleanedHeaders = removeSentryInternalHeaders(request.headers).toMutableList()
 
-        TracingUtils.traceIfAllowed(scopes, request.url, request.headers.filter { it.name == BaggageHeader.BAGGAGE_HEADER }.map { it.value }, span)?.let {
-            cleanedHeaders.add(HttpHeader(it.sentryTraceHeader.name, it.sentryTraceHeader.value))
-            it.baggageHeader?.let { baggageHeader ->
-                cleanedHeaders = cleanedHeaders.filterNot { it.name == BaggageHeader.BAGGAGE_HEADER }.toMutableList().apply {
-                    add(HttpHeader(baggageHeader.name, baggageHeader.value))
+        if (!isIgnored()) {
+            TracingUtils.traceIfAllowed(scopes, request.url, request.headers.filter { it.name == BaggageHeader.BAGGAGE_HEADER }.map { it.value }, span)?.let {
+                cleanedHeaders.add(HttpHeader(it.sentryTraceHeader.name, it.sentryTraceHeader.value))
+                it.baggageHeader?.let { baggageHeader ->
+                    cleanedHeaders = cleanedHeaders.filterNot { it.name == BaggageHeader.BAGGAGE_HEADER }.toMutableList().apply {
+                        add(HttpHeader(baggageHeader.name, baggageHeader.value))
+                    }
                 }
             }
         }
@@ -136,6 +139,10 @@ class SentryApollo3HttpInterceptor @JvmOverloads constructor(
         return requestBuilder.build()
     }
 
+    private fun isIgnored(): Boolean {
+        return SpanUtils.isIgnored(scopes.getOptions().getIgnoredSpanOrigins(), TRACE_ORIGIN)
+    }
+
     private fun removeSentryInternalHeaders(headers: List<HttpHeader>): List<HttpHeader> {
         return headers.filterNot {
             it.name.equals(SENTRY_APOLLO_3_VARIABLES, true) ||

sentry-apollo-3/src/test/java/io/sentry/apollo3/SentryApollo3InterceptorTest.kt

@@ -208,6 +208,16 @@ class SentryApollo3InterceptorTest {
         assertNotNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
     }
 
+    @Test
+    fun `does not add sentry-trace header when span origin is ignored`() {
+        fixture.options.ignoredSpanOrigins = listOf("auto.graphql.apollo3")
+        executeQuery(isSpanActive = false)
+
+        val recorderRequest = fixture.server.takeRequest(mockServerRequestTimeoutMillis, TimeUnit.MILLISECONDS)!!
+        assertNull(recorderRequest.headers[SentryTraceHeader.SENTRY_TRACE_HEADER])
+        assertNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
+    }
+
     @Test
     fun `when there is an active span, adds sentry trace headers to the request`() {
         executeQuery()

sentry-apollo/src/main/java/io/sentry/apollo/SentryApolloInterceptor.kt

@@ -25,6 +25,7 @@ import io.sentry.SpanStatus
 import io.sentry.TypeCheckHint.APOLLO_REQUEST
 import io.sentry.TypeCheckHint.APOLLO_RESPONSE
 import io.sentry.util.IntegrationUtils.addIntegrationToSdkVersion
+import io.sentry.util.SpanUtils
 import io.sentry.util.TracingUtils
 import java.util.Locale
 import java.util.concurrent.Executor
@@ -115,7 +116,7 @@ class SentryApolloInterceptor(
     private fun addTracingHeaders(request: InterceptorRequest, span: ISpan?): RequestHeaders {
         val requestHeaderBuilder = request.requestHeaders.toBuilder()
 
-        if (scopes.options.isTraceSampling) {
+        if (scopes.options.isTraceSampling && !isIgnored()) {
             // we have no access to URI, no way to verify tracing origins
             TracingUtils.trace(
                 scopes,
@@ -135,6 +136,10 @@ class SentryApolloInterceptor(
         return requestHeaderBuilder.build()
     }
 
+    private fun isIgnored(): Boolean {
+        return SpanUtils.isIgnored(scopes.getOptions().getIgnoredSpanOrigins(), TRACE_ORIGIN)
+    }
+
     private fun startChild(request: InterceptorRequest, activeSpan: ISpan): ISpan {
         val operation = request.operation.name().name()
         val operationType = when (request.operation) {

sentry-apollo/src/test/java/io/sentry/apollo/SentryApolloInterceptorTest.kt

@@ -161,6 +161,16 @@ class SentryApolloInterceptorTest {
         assertNotNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
     }
 
+    @Test
+    fun `does not add sentry-trace header when span origin is ignored`() {
+        fixture.options.ignoredSpanOrigins = listOf("auto.graphql.apollo")
+        executeQuery(isSpanActive = false)
+
+        val recorderRequest = fixture.server.takeRequest(mockServerRequestTimeoutMillis, TimeUnit.MILLISECONDS)!!
+        assertNull(recorderRequest.headers[SentryTraceHeader.SENTRY_TRACE_HEADER])
+        assertNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
+    }
+
     @Test
     fun `when there is an active span, adds sentry trace headers to the request`() {
         executeQuery()

sentry-okhttp/src/main/java/io/sentry/okhttp/SentryOkHttpInterceptor.kt

@@ -18,6 +18,7 @@ import io.sentry.transport.CurrentDateProvider
 import io.sentry.util.IntegrationUtils.addIntegrationToSdkVersion
 import io.sentry.util.Platform
 import io.sentry.util.PropagationTargetsUtils
+import io.sentry.util.SpanUtils
 import io.sentry.util.TracingUtils
 import io.sentry.util.UrlUtils
 import okhttp3.Interceptor
@@ -93,16 +94,21 @@ public open class SentryOkHttpInterceptor(
         try {
             val requestBuilder = request.newBuilder()
 
-            TracingUtils.traceIfAllowed(
-                scopes,
-                request.url.toString(),
-                request.headers(BaggageHeader.BAGGAGE_HEADER),
-                span
-            )?.let { tracingHeaders ->
-                requestBuilder.addHeader(tracingHeaders.sentryTraceHeader.name, tracingHeaders.sentryTraceHeader.value)
-                tracingHeaders.baggageHeader?.let {
-                    requestBuilder.removeHeader(BaggageHeader.BAGGAGE_HEADER)
-                    requestBuilder.addHeader(it.name, it.value)
+            if (!isIgnored()) {
+                TracingUtils.traceIfAllowed(
+                    scopes,
+                    request.url.toString(),
+                    request.headers(BaggageHeader.BAGGAGE_HEADER),
+                    span
+                )?.let { tracingHeaders ->
+                    requestBuilder.addHeader(
+                        tracingHeaders.sentryTraceHeader.name,
+                        tracingHeaders.sentryTraceHeader.value
+                    )
+                    tracingHeaders.baggageHeader?.let {
+                        requestBuilder.removeHeader(BaggageHeader.BAGGAGE_HEADER)
+                        requestBuilder.addHeader(it.name, it.value)
+                    }
                 }
             }
 
@@ -144,6 +150,10 @@ public open class SentryOkHttpInterceptor(
         }
     }
 
+    private fun isIgnored(): Boolean {
+        return SpanUtils.isIgnored(scopes.getOptions().getIgnoredSpanOrigins(), TRACE_ORIGIN)
+    }
+
     private fun sendBreadcrumb(
         request: Request,
         code: Int?,

sentry-okhttp/src/test/java/io/sentry/okhttp/SentryOkHttpInterceptorTest.kt

@@ -10,6 +10,7 @@ import io.sentry.IScope
 import io.sentry.IScopes
 import io.sentry.Scope
 import io.sentry.ScopeCallback
+import io.sentry.Sentry
 import io.sentry.SentryOptions
 import io.sentry.SentryTraceHeader
 import io.sentry.SentryTracer
@@ -73,16 +74,18 @@ class SentryOkHttpInterceptorTest {
                     HttpStatusCodeRange.DEFAULT_MAX
                 )
             ),
-            sendDefaultPii: Boolean = false
+            sendDefaultPii: Boolean = false,
+            optionsConfiguration: Sentry.OptionsConfiguration<SentryOptions>? = null
         ): OkHttpClient {
-            options = SentryOptions().apply {
-                dsn = "https://[email protected]/proj"
+            options = SentryOptions().also {
+                optionsConfiguration?.configure(it)
+                it.dsn = "https://[email protected]/proj"
                 if (includeMockServerInTracePropagationTargets) {
-                    setTracePropagationTargets(listOf(server.hostName))
+                    it.setTracePropagationTargets(listOf(server.hostName))
                 } else if (!keepDefaultTracePropagationTargets) {
-                    setTracePropagationTargets(listOf("other-api"))
+                    it.setTracePropagationTargets(listOf("other-api"))
                 }
-                isSendDefaultPii = sendDefaultPii
+                it.isSendDefaultPii = sendDefaultPii
             }
             scope = Scope(options)
             whenever(scopes.options).thenReturn(options)
@@ -207,6 +210,17 @@ class SentryOkHttpInterceptorTest {
         assertNotNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
     }
 
+    @Test
+    fun `does not add sentry-trace header when span origin is ignored`() {
+        val sut = fixture.getSut(isSpanActive = false) { options ->
+            options.ignoredSpanOrigins = listOf("auto.http.okhttp")
+        }
+        sut.newCall(getRequest()).execute()
+        val recorderRequest = fixture.server.takeRequest(mockServerRequestTimeoutMillis, TimeUnit.MILLISECONDS)!!
+        assertNull(recorderRequest.headers[SentryTraceHeader.SENTRY_TRACE_HEADER])
+        assertNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
+    }
+
     @Test
     fun `when there is no active span and host if not allowed, does not add sentry trace header to the request`() {
         val sut = fixture.getSut(isSpanActive = false)

sentry-openfeign/src/main/java/io/sentry/openfeign/SentryFeignClient.java

@@ -15,6 +15,7 @@
 import io.sentry.SpanOptions;
 import io.sentry.SpanStatus;
 import io.sentry.util.Objects;
+import io.sentry.util.SpanUtils;
 import io.sentry.util.TracingUtils;
 import io.sentry.util.UrlUtils;
 import java.io.IOException;
@@ -98,6 +99,10 @@ public Response execute(final @NotNull Request request, final @NotNull Request.O
 
   private @NotNull Request maybeAddTracingHeaders(
       final @NotNull Request request, final @Nullable ISpan span) {
+    if (isIgnored()) {
+      return request;
+    }
+
     final @NotNull RequestWrapper requestWrapper = new RequestWrapper(request);
     final @Nullable Collection<String> requestBaggageHeaders =
         request.headers().get(BaggageHeader.BAGGAGE_HEADER);
@@ -124,6 +129,10 @@ public Response execute(final @NotNull Request request, final @NotNull Request.O
     return requestWrapper.build();
   }
 
+  private boolean isIgnored() {
+    return SpanUtils.isIgnored(scopes.getOptions().getIgnoredSpanOrigins(), TRACE_ORIGIN);
+  }
+
   private void addBreadcrumb(final @NotNull Request request, final @Nullable Response response) {
     final Breadcrumb breadcrumb =
         Breadcrumb.http(

sentry-openfeign/src/test/kotlin/io/sentry/openfeign/SentryFeignClientTest.kt

@@ -130,6 +130,17 @@ class SentryFeignClientTest {
         assertNotNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
     }
 
+    @Test
+    fun `does not add sentry trace header when span origin is ignored`() {
+        fixture.sentryOptions.dsn = "https://[email protected]/proj"
+        fixture.sentryOptions.ignoredSpanOrigins = listOf("auto.http.openfeign")
+        val sut = fixture.getSut(isSpanActive = false)
+        sut.getOk()
+        val recorderRequest = fixture.server.takeRequest(mockServerRequestTimeoutMillis, TimeUnit.MILLISECONDS)!!
+        assertNull(recorderRequest.headers[SentryTraceHeader.SENTRY_TRACE_HEADER])
+        assertNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
+    }
+
     @Test
     fun `when there is no active span, does not add sentry trace header to the request if host is disallowed`() {
         fixture.sentryOptions.setTracePropagationTargets(listOf("some-host-that-does-not-exist"))

sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/SentrySpanRestClientCustomizerTest.kt

@@ -17,6 +17,7 @@ import okhttp3.mockwebserver.MockWebServer
 import okhttp3.mockwebserver.SocketPolicy
 import org.apache.hc.client5.http.impl.classic.HttpClients
 import org.assertj.core.api.Assertions.assertThat
+import org.junit.Assert.assertNull
 import org.mockito.Mockito.doAnswer
 import org.mockito.Mockito.mock
 import org.mockito.Mockito.verify
@@ -248,6 +249,24 @@ class SentrySpanRestClientCustomizerTest {
         assertTrue(baggageHeaderValues[0].contains("sentry-trace_id"))
     }
 
+    @Test
+    fun `does not add sentry-trace header if span origin is ignored`() {
+        fixture.sentryOptions.ignoredSpanOrigins = listOf("auto.http.spring_jakarta.restclient")
+        val sut = fixture.getSut(isTransactionActive = false)
+        val headers = HttpHeaders()
+
+        sut.build()
+            .get()
+            .uri(fixture.url)
+            .httpRequest { it.headers.addAll(headers) }
+            .retrieve()
+            .toEntity(String::class.java)
+
+        val recorderRequest = fixture.mockServer.takeRequest(mockServerRequestTimeoutMillis, TimeUnit.MILLISECONDS)!!
+        assertNull(recorderRequest.headers[SentryTraceHeader.SENTRY_TRACE_HEADER])
+        assertNull(recorderRequest.headers[BaggageHeader.BAGGAGE_HEADER])
+    }
+
     @Test
     fun `when transaction is active adds breadcrumb when http calls succeeds`() {
         fixture.getSut(isTransactionActive = true)