feat(debugger): add Remote Enablement support (#7137)

Implements Remote Enablement (RE) for Dynamic Instrumentation/Live Debugger and
Code Origin via APM_TRACING_MULTICONFIG.

Key Remote Config changes:
- Add RCClientLibConfigManager class for priority-based merging of multiple
  APM_TRACING configs (Service+Env > Service > Env > Cluster > Org)
- Add APM_TRACING_MULTICONFIG, APM_TRACING_ENABLE_DYNAMIC_INSTRUMENTATION,
  APM_TRACING_ENABLE_CODE_ORIGIN, and APM_TRACING_ENABLE_LIVE_DEBUGGING
  capability flags
- Centralize client library configuration remote config logic in
  config/remote_config.js

Key Debugger changes:
- Add lifecycle methods (isStarted(), stop(), configure()) to debugger/index.js
  for dynamic control

Key Code Origin changes:
- Add runtime enable/disable support for Code Origin via Remote Config
- Update Express and Fastify code origin plugins to pre-compute entry tags at
  route registration time when RC is enabled (allowing runtime enabling to work
  immediately)
- Add integration tests for Code Origin Remote Config toggling

Author: watson

integration-tests/code-origin-remote-config.spec.js

@@ -0,0 +1,174 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+const path = require('node:path')
+const Axios = require('axios')
+const { FakeAgent, sandboxCwd, useSandbox, spawnProc } = require('./helpers')
+
+const ACKNOWLEDGED = 2
+
+describe('Code Origin Remote Config', function () {
+  this.timeout(20000)
+
+  let cwd, agent, proc, axios
+
+  useSandbox(
+    ['express', 'fastify'],
+    false,
+    [path.join(__dirname, 'code-origin')]
+  )
+
+  before(() => {
+    cwd = sandboxCwd()
+  })
+
+  afterEach(async () => {
+    proc?.kill()
+    await agent?.stop()
+  })
+
+  const frameworks = [
+    { name: 'Express', spanName: 'express.request', appFile: 'express-app.js' },
+    { name: 'Fastify', spanName: 'fastify.request', appFile: 'fastify-app.js' }
+  ]
+
+  const setupApp = (framework, envVars) => async () => {
+    const appFile = path.join(cwd, 'code-origin', framework.appFile)
+    agent = await new FakeAgent().start()
+    proc = await spawnProc(appFile, {
+      cwd,
+      env: {
+        DD_TRACE_AGENT_PORT: agent.port,
+        DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS: 0.1,
+        ...envVars
+      }
+    })
+    axios = Axios.create({
+      baseURL: proc.url,
+      headers: { 'Content-Type': 'application/json' }
+    })
+  }
+
+  const addRemoteConfigAndWaitForAck = (libConfig) => {
+    return /** @type {Promise<void>} */ (new Promise((resolve) => {
+      // Random config id - Just needs to be unique between calls to this function
+      const configId = Math.random().toString(36).slice(2)
+      const handler = (id, version, state) => {
+        if (id === configId && state === ACKNOWLEDGED) {
+          agent.removeListener('remote-config-ack-update', handler)
+          resolve()
+        }
+      }
+      agent.on('remote-config-ack-update', handler)
+      agent.addRemoteConfig({
+        product: 'APM_TRACING',
+        id: configId,
+        config: {
+          service_target: { service: 'node', env: '*' },
+          lib_config: libConfig
+        }
+      })
+    }))
+  }
+
+  const assertCodeOriginPresent = (framework, url = '/hello') => {
+    return Promise.all([
+      agent.assertMessageReceived(({ payload }) => {
+        const spans = payload.flatMap(p => p)
+        const requestSpan = spans.find(span => span.name === framework.spanName)
+        assert.ok(requestSpan, `${framework.spanName} span should exist`)
+        assert.strictEqual(requestSpan.meta['_dd.code_origin.type'], 'entry')
+        assert.ok(requestSpan.meta['_dd.code_origin.frames.0.file'])
+        assert.ok(requestSpan.meta['_dd.code_origin.frames.0.line'])
+      }, 3000),
+      axios.get(url)
+    ])
+  }
+
+  const assertCodeOriginAbsent = (framework, url = '/hello') => {
+    return Promise.all([
+      agent.assertMessageReceived(({ payload }) => {
+        const spans = payload.flatMap(p => p)
+        const requestSpan = spans.find(span => span.name === framework.spanName)
+        assert.ok(requestSpan, `${framework.spanName} span should exist`)
+        assert.strictEqual(requestSpan.meta['_dd.code_origin.type'], undefined)
+        assert.strictEqual(requestSpan.meta['_dd.code_origin.frames.0.file'], undefined)
+      }, 3000),
+      axios.get(url)
+    ])
+  }
+
+  frameworks.forEach(framework => {
+    describe(framework.name, () => {
+      describe('both CO and RC enabled at boot (runtime disable)', () => {
+        beforeEach(setupApp(framework, {
+          DD_CODE_ORIGIN_FOR_SPANS_ENABLED: 'true',
+          DD_REMOTE_CONFIG_ENABLED: 'true'
+        }))
+
+        it('should disable code origin tags at runtime via remote config', async () => {
+          // Step 1: Make a request with code origin enabled (default)
+          await assertCodeOriginPresent(framework)
+
+          // Verify config shows enabled
+          const configBefore = await axios.get('/config')
+          assert.strictEqual(configBefore.data.codeOriginEnabled, true)
+          assert.strictEqual(configBefore.data.remoteConfigEnabled, true)
+
+          // Step 2: Disable code origin via remote config
+          await addRemoteConfigAndWaitForAck({ code_origin_enabled: false })
+
+          // Verify config shows disabled
+          const configAfter = await axios.get('/config')
+          assert.strictEqual(configAfter.data.codeOriginEnabled, false)
+          assert.strictEqual(configAfter.data.remoteConfigEnabled, true)
+
+          // Step 3: Make another request and verify NO code origin tags
+          // The tags are pre-computed and cached, but not applied since _enabled is false
+          await assertCodeOriginAbsent(framework)
+        })
+      })
+
+      describe('CO enabled at boot, RC disabled', () => {
+        beforeEach(setupApp(framework, {
+          DD_CODE_ORIGIN_FOR_SPANS_ENABLED: 'true',
+          DD_REMOTE_CONFIG_ENABLED: 'false'
+        }))
+
+        it('should pre-compute and add code origin tags', async () => {
+          // With CO enabled, tags should be computed and added normally
+          await assertCodeOriginPresent(framework)
+        })
+      })
+
+      describe('RC enabled at boot, CO disabled', () => {
+        beforeEach(setupApp(framework, {
+          DD_CODE_ORIGIN_FOR_SPANS_ENABLED: 'false',
+          DD_REMOTE_CONFIG_ENABLED: 'true'
+        }))
+
+        it('should pre-compute but not add code origin tags, then add them after runtime enable', async () => {
+          // Step 1: With RC enabled but CO disabled, tags are pre-computed (for potential runtime enabling)
+          // but should NOT be applied to spans since CO is disabled
+          await assertCodeOriginAbsent(framework)
+
+          // Verify config shows CO disabled
+          const configBefore = await axios.get('/config')
+          assert.strictEqual(configBefore.data.codeOriginEnabled, false)
+          assert.strictEqual(configBefore.data.remoteConfigEnabled, true)
+
+          // Step 2: Enable code origin at runtime via remote config
+          await addRemoteConfigAndWaitForAck({ code_origin_enabled: true })
+
+          // Verify config shows CO enabled
+          const configAfter = await axios.get('/config')
+          assert.strictEqual(configAfter.data.codeOriginEnabled, true)
+
+          // Step 3: Make another request and verify code origin tags ARE now present
+          // This works because tags were pre-computed at boot when RC was enabled
+          await assertCodeOriginPresent(framework)
+        })
+      })
+    })
+  })
+})

integration-tests/code-origin/express-app.js

@@ -0,0 +1,25 @@
+'use strict'
+
+// @ts-expect-error This code is running in a sandbox where dd-trace is available
+const tracer = require('dd-trace')
+
+tracer.init({ flushInterval: 1 })
+
+const express = require('express')
+const app = express()
+
+app.get('/hello', (req, res) => {
+  res.json({ message: 'Hello World' })
+})
+
+app.get('/config', (req, res) => {
+  const config = tracer._tracer._config
+  res.json({
+    codeOriginEnabled: config.codeOriginForSpans.enabled,
+    remoteConfigEnabled: config.remoteConfig.enabled
+  })
+})
+
+const server = app.listen(process.env.APP_PORT || 0, () => {
+  process.send?.({ port: (/** @type {import('net').AddressInfo} */ (server.address())).port })
+})

integration-tests/code-origin/fastify-app.js

@@ -0,0 +1,29 @@
+'use strict'
+
+// @ts-expect-error This code is running in a sandbox where dd-trace is available
+const tracer = require('dd-trace')
+
+tracer.init({ flushInterval: 1 })
+
+// @ts-expect-error This code is running in a sandbox where fastify is available
+const fastify = require('fastify')
+const app = fastify()
+
+app.get('/hello', (req, res) => {
+  res.send({ message: 'Hello World' })
+})
+
+app.get('/config', (req, res) => {
+  const config = tracer._tracer._config
+  res.send({
+    codeOriginEnabled: config.codeOriginForSpans.enabled,
+    remoteConfigEnabled: config.remoteConfig.enabled
+  })
+})
+
+app.listen({ port: process.env.APP_PORT || 0 }, (error) => {
+  if (error) {
+    throw error
+  }
+  process.send?.({ port: app.server.address().port })
+})

packages/datadog-plugin-express/src/code_origin.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const dc = require('dc-polyfill')
+
 const { entryTags } = require('../../datadog-code-origin')
 const Plugin = require('../../dd-trace/src/plugins/plugin')
 const web = require('../../dd-trace/src/plugins/util/web')
@@ -12,29 +14,33 @@ class ExpressCodeOriginForSpansPlugin extends Plugin {
 
     const layerTags = new WeakMap()
 
-    this.addSub('apm:express:middleware:enter', ({ req, layer }) => {
+    // Middleware/request handling: apply pre-computed tags to spans
+    const handleMiddlewareEnter = ({ req, layer }) => {
       const tags = layerTags.get(layer)
       if (!tags) return
       web.getContext(req)?.span?.addTags(tags)
-    })
-
-    this.addSub('apm:express:route:added', ({ topOfStackFunc, layer }) => {
-      if (!layer) return
-      if (layerTags.has(layer)) return
-      layerTags.set(layer, entryTags(topOfStackFunc))
-    })
+    }
 
-    this.addSub('apm:router:middleware:enter', ({ req, layer }) => {
-      const tags = layerTags.get(layer)
-      if (!tags) return
-      web.getContext(req)?.span?.addTags(tags)
-    })
+    this.addSub('apm:express:middleware:enter', handleMiddlewareEnter)
+    this.addSub('apm:router:middleware:enter', handleMiddlewareEnter)
 
-    this.addSub('apm:router:route:added', ({ topOfStackFunc, layer }) => {
+    // Route added handling: compute and cache tags
+    const handleRouteAdded = ({ topOfStackFunc, layer }) => {
       if (!layer) return
       if (layerTags.has(layer)) return
       layerTags.set(layer, entryTags(topOfStackFunc))
-    })
+    }
+
+    if (this._tracerConfig.remoteConfig?.enabled) {
+      // When RC is enabled, use manual subscriptions (always pre-compute)
+      // This allows tags to be computed even when CO is disabled, so runtime enabling works
+      dc.channel('apm:express:route:added').subscribe(handleRouteAdded)
+      dc.channel('apm:router:route:added').subscribe(handleRouteAdded)
+    } else {
+      // When RC is disabled, use addSub (only computes when CO is enabled)
+      this.addSub('apm:express:route:added', handleRouteAdded)
+      this.addSub('apm:router:route:added', handleRouteAdded)
+    }
   }
 }
 

packages/datadog-plugin-fastify/src/code_origin.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const dc = require('dc-polyfill')
+
 const { entryTags } = require('../../datadog-code-origin')
 const Plugin = require('../../dd-trace/src/plugins/plugin')
 const web = require('../../dd-trace/src/plugins/util/web')
@@ -18,11 +20,22 @@ class FastifyCodeOriginForSpansPlugin extends Plugin {
       web.getContext(req)?.span?.addTags(tags)
     })
 
-    this.addSub('apm:fastify:route:added', ({ routeOptions, onRoute }) => {
-      if (!routeOptions.config) routeOptions.config = {}
-      routeOptions.config[kCodeOriginForSpansTagsSym] = entryTags(onRoute)
-    })
+    if (this._tracerConfig.remoteConfig?.enabled) {
+      // When RC is enabled, use manual subscription (always pre-computes)
+      // This allows tags to be computed even when CO is disabled, so runtime enabling works
+      dc.channel('apm:fastify:route:added').subscribe(handleRouteAdded)
+    } else {
+      // When RC is disabled, use addSub (only computes when CO is enabled)
+      this.addSub('apm:fastify:route:added', handleRouteAdded)
+    }
   }
 }
 
 module.exports = FastifyCodeOriginForSpansPlugin
+
+// Route added handling: compute and cache tags
+function handleRouteAdded ({ routeOptions, onRoute }) {
+  if (!routeOptions.config) routeOptions.config = {}
+  if (routeOptions.config[kCodeOriginForSpansTagsSym]) return
+  routeOptions.config[kCodeOriginForSpansTagsSym] = entryTags(onRoute)
+}