From 942530af60f40eaf10606601ddfab022c6e75ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 19 Mar 2025 14:32:11 +0100 Subject: [PATCH 01/32] [test optimization] Fix playwright e2e tests flakiness (#5438) --- integration-tests/playwright/playwright.spec.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 7dfb3ee3fe6..69bd83ca123 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -47,12 +47,14 @@ versions.forEach((version) => { let sandbox, cwd, receiver, childProcess, webAppPort before(async function () { - // bump from 30 to 60 seconds because playwright dependencies are heavy - this.timeout(60000) + // bump from 60 to 90 seconds because we also need to install dependencies and download chromium + this.timeout(90000) sandbox = await createSandbox([`@playwright/test@${version}`, 'typescript'], true) cwd = sandbox.folder - // install necessary browser const { NODE_OPTIONS, ...restOfEnv } = process.env + // Install system dependencies + execSync('npx playwright install-deps', { cwd, env: restOfEnv }) + // Install chromium (configured in integration-tests/playwright.config.js) execSync('npx playwright install', { cwd, env: restOfEnv }) webAppPort = await getPort() webAppServer.listen(webAppPort) From 9461c3ad458519738feefc65463a4cee062c366a Mon Sep 17 00:00:00 2001 From: Ilyas Shabi Date: Wed, 19 Mar 2025 15:38:56 +0100 Subject: [PATCH 02/32] Report waf init and update success and failure (#5388) * Report waf init and update success * handle waf init failure * do not report ecent rules when success fails * remove default value of diagnosticsRules * use default args to report init --- packages/dd-trace/src/appsec/reporter.js | 18 ++--- .../dd-trace/src/appsec/telemetry/index.js | 8 +-- packages/dd-trace/src/appsec/telemetry/waf.js | 14 ++-- .../dd-trace/src/appsec/waf/waf_manager.js | 23 ++++-- packages/dd-trace/test/appsec/index.spec.js | 2 +- .../dd-trace/test/appsec/reporter.spec.js | 23 ++++-- .../test/appsec/telemetry/waf.spec.js | 62 ++++++++++++---- .../dd-trace/test/appsec/waf/index.spec.js | 72 ++++++++++++++----- .../test/appsec/waf/waf_manager.spec.js | 2 +- 9 files changed, 163 insertions(+), 61 deletions(-) diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index 9a0b7467c37..c5f3bdce56c 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -88,16 +88,18 @@ function formatHeaderName (name) { .toLowerCase() } -function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) { - metricsQueue.set('_dd.appsec.waf.version', wafVersion) - - metricsQueue.set('_dd.appsec.event_rules.loaded', diagnosticsRules.loaded?.length || 0) - metricsQueue.set('_dd.appsec.event_rules.error_count', diagnosticsRules.failed?.length || 0) - if (diagnosticsRules.failed?.length) { - metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors)) +function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}, success = false) { + if (success) { + metricsQueue.set('_dd.appsec.waf.version', wafVersion) + + metricsQueue.set('_dd.appsec.event_rules.loaded', diagnosticsRules.loaded?.length || 0) + metricsQueue.set('_dd.appsec.event_rules.error_count', diagnosticsRules.failed?.length || 0) + if (diagnosticsRules.failed?.length) { + metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors)) + } } - incrementWafInitMetric(wafVersion, rulesVersion) + incrementWafInitMetric(wafVersion, rulesVersion, success) } function reportMetrics (metrics, raspRule) { diff --git a/packages/dd-trace/src/appsec/telemetry/index.js b/packages/dd-trace/src/appsec/telemetry/index.js index 1a0141c2068..0593efd97b9 100644 --- a/packages/dd-trace/src/appsec/telemetry/index.js +++ b/packages/dd-trace/src/appsec/telemetry/index.js @@ -74,16 +74,16 @@ function updateWafRequestsMetricTags (metrics, req) { return trackWafMetrics(store, metrics) } -function incrementWafInitMetric (wafVersion, rulesVersion) { +function incrementWafInitMetric (wafVersion, rulesVersion, success) { if (!enabled) return - incrementWafInit(wafVersion, rulesVersion) + incrementWafInit(wafVersion, rulesVersion, success) } -function incrementWafUpdatesMetric (wafVersion, rulesVersion) { +function incrementWafUpdatesMetric (wafVersion, rulesVersion, success) { if (!enabled) return - incrementWafUpdates(wafVersion, rulesVersion) + incrementWafUpdates(wafVersion, rulesVersion, success) } function incrementWafRequestsMetric (req) { diff --git a/packages/dd-trace/src/appsec/telemetry/waf.js b/packages/dd-trace/src/appsec/telemetry/waf.js index c79ed04c243..bf995741ce2 100644 --- a/packages/dd-trace/src/appsec/telemetry/waf.js +++ b/packages/dd-trace/src/appsec/telemetry/waf.js @@ -91,16 +91,22 @@ function getOrCreateMetricTags (store, versionsTags) { return metricTags } -function incrementWafInit (wafVersion, rulesVersion) { +function incrementWafInit (wafVersion, rulesVersion, success) { const versionsTags = getVersionsTags(wafVersion, rulesVersion) + appsecMetrics.count('waf.init', { ...versionsTags, success }).inc() - appsecMetrics.count('waf.init', versionsTags).inc() + if (!success) { + appsecMetrics.count('waf.config_errors', versionsTags).inc() + } } -function incrementWafUpdates (wafVersion, rulesVersion) { +function incrementWafUpdates (wafVersion, rulesVersion, success) { const versionsTags = getVersionsTags(wafVersion, rulesVersion) + appsecMetrics.count('waf.updates', { ...versionsTags, success }).inc() - appsecMetrics.count('waf.updates', versionsTags).inc() + if (!success) { + appsecMetrics.count('waf.config_errors', versionsTags).inc() + } } function incrementWafRequests (store) { diff --git a/packages/dd-trace/src/appsec/waf/waf_manager.js b/packages/dd-trace/src/appsec/waf/waf_manager.js index 520438d8a20..e745818aa9e 100644 --- a/packages/dd-trace/src/appsec/waf/waf_manager.js +++ b/packages/dd-trace/src/appsec/waf/waf_manager.js @@ -11,20 +11,23 @@ class WAFManager { this.config = config this.wafTimeout = config.wafTimeout this.ddwaf = this._loadDDWAF(rules) - this.ddwafVersion = this.ddwaf.constructor.version() this.rulesVersion = this.ddwaf.diagnostics.ruleset_version - Reporter.reportWafInit(this.ddwafVersion, this.rulesVersion, this.ddwaf.diagnostics.rules) + Reporter.reportWafInit(this.ddwafVersion, this.rulesVersion, this.ddwaf.diagnostics.rules, true) } _loadDDWAF (rules) { try { // require in `try/catch` because this can throw at require time const { DDWAF } = require('@datadog/native-appsec') + this.ddwafVersion = DDWAF.version() const { obfuscatorKeyRegex, obfuscatorValueRegex } = this.config return new DDWAF(rules, { obfuscatorKeyRegex, obfuscatorValueRegex }) } catch (err) { + this.ddwafVersion = this.ddwafVersion || 'unknown' + Reporter.reportWafInit(this.ddwafVersion, 'unknown') + log.error('[ASM] AppSec could not load native package. In-app WAF features will not be available.') throw err @@ -49,13 +52,19 @@ class WAFManager { } update (newRules) { - this.ddwaf.update(newRules) + try { + this.ddwaf.update(newRules) - if (this.ddwaf.diagnostics.ruleset_version) { - this.rulesVersion = this.ddwaf.diagnostics.ruleset_version - } + if (this.ddwaf.diagnostics.ruleset_version) { + this.rulesVersion = this.ddwaf.diagnostics.ruleset_version + } - Reporter.reportWafUpdate(this.ddwafVersion, this.rulesVersion) + Reporter.reportWafUpdate(this.ddwafVersion, this.rulesVersion, true) + } catch (error) { + Reporter.reportWafUpdate(this.ddwafVersion, 'unknown', false) + + throw error + } } destroy () { diff --git a/packages/dd-trace/test/appsec/index.spec.js b/packages/dd-trace/test/appsec/index.spec.js index 19f115a7340..88ffa386d9c 100644 --- a/packages/dd-trace/test/appsec/index.spec.js +++ b/packages/dd-trace/test/appsec/index.spec.js @@ -646,7 +646,7 @@ describe('AppSec Index', function () { responseBody.publish({ req, res, body }) expect(apiSecuritySampler.sampleRequest).to.have.been.calledOnceWith(req, res) - expect(waf.run).to.been.calledOnceWith({ + expect(waf.run).to.have.been.calledOnceWith({ persistent: { [addresses.HTTP_INCOMING_RESPONSE_BODY]: body } diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index 271a81e2725..887f620b948 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -101,7 +101,7 @@ describe('reporter', () => { } it('should add some entries to metricsQueue', () => { - Reporter.reportWafInit(wafVersion, rulesVersion, diagnosticsRules) + Reporter.reportWafInit(wafVersion, rulesVersion, diagnosticsRules, true) expect(Reporter.metricsQueue.get('_dd.appsec.waf.version')).to.be.eq(wafVersion) expect(Reporter.metricsQueue.get('_dd.appsec.event_rules.loaded')).to.be.eq(3) @@ -110,10 +110,19 @@ describe('reporter', () => { .to.be.eq(JSON.stringify(diagnosticsRules.errors)) }) + it('should not add entries to metricsQueue with success false', () => { + Reporter.reportWafInit(wafVersion, rulesVersion, diagnosticsRules, false) + + expect(Reporter.metricsQueue.get('_dd.appsec.waf.version')).to.be.undefined + expect(Reporter.metricsQueue.get('_dd.appsec.event_rules.loaded')).to.be.undefined + expect(Reporter.metricsQueue.get('_dd.appsec.event_rules.error_count')).to.be.undefined + expect(Reporter.metricsQueue.get('_dd.appsec.event_rules.errors')).to.be.undefined + }) + it('should call incrementWafInitMetric', () => { - Reporter.reportWafInit(wafVersion, rulesVersion, diagnosticsRules) + Reporter.reportWafInit(wafVersion, rulesVersion, diagnosticsRules, true) - expect(telemetry.incrementWafInitMetric).to.have.been.calledOnceWithExactly(wafVersion, rulesVersion) + expect(telemetry.incrementWafInitMetric).to.have.been.calledOnceWithExactly(wafVersion, rulesVersion, true) }) it('should not fail with undefined arguments', () => { @@ -121,12 +130,12 @@ describe('reporter', () => { const rulesVersion = undefined const diagnosticsRules = undefined - Reporter.reportWafInit(wafVersion, rulesVersion, diagnosticsRules) + Reporter.reportWafInit(wafVersion, rulesVersion, diagnosticsRules, true) expect(Reporter.metricsQueue.get('_dd.appsec.event_rules.loaded')).to.be.eq(0) expect(Reporter.metricsQueue.get('_dd.appsec.event_rules.error_count')).to.be.eq(0) - expect(telemetry.incrementWafInitMetric).to.have.been.calledOnceWithExactly(wafVersion, rulesVersion) + expect(telemetry.incrementWafInitMetric).to.have.been.calledOnceWithExactly(wafVersion, rulesVersion, true) }) }) @@ -380,9 +389,9 @@ describe('reporter', () => { describe('reportWafUpdate', () => { it('should call incrementWafUpdatesMetric', () => { - Reporter.reportWafUpdate('0.0.1', '0.0.2') + Reporter.reportWafUpdate('0.0.1', '0.0.2', true) - expect(telemetry.incrementWafUpdatesMetric).to.have.been.calledOnceWithExactly('0.0.1', '0.0.2') + expect(telemetry.incrementWafUpdatesMetric).to.have.been.calledOnceWithExactly('0.0.1', '0.0.2', true) }) }) diff --git a/packages/dd-trace/test/appsec/telemetry/waf.spec.js b/packages/dd-trace/test/appsec/telemetry/waf.spec.js index 4962d7fccf2..eff86ddabb6 100644 --- a/packages/dd-trace/test/appsec/telemetry/waf.spec.js +++ b/packages/dd-trace/test/appsec/telemetry/waf.spec.js @@ -185,11 +185,12 @@ describe('Appsec Waf Telemetry metrics', () => { describe('incWafInitMetric', () => { it('should increment waf.init metric', () => { - appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion) + appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion, true) expect(count).to.have.been.calledOnceWithExactly('waf.init', { waf_version: wafVersion, - event_rules_version: rulesVersion + event_rules_version: rulesVersion, + success: true }) expect(inc).to.have.been.calledOnce }) @@ -197,9 +198,9 @@ describe('Appsec Waf Telemetry metrics', () => { it('should increment waf.init metric multiple times', () => { sinon.restore() - appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion) - appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion) - appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion) + appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion, true) + appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion, true) + appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion, true) const { metrics } = appsecNamespace.toJSON() expect(metrics.series.length).to.be.eq(1) @@ -208,16 +209,35 @@ describe('Appsec Waf Telemetry metrics', () => { expect(metrics.series[0].points[0][1]).to.be.eq(3) expect(metrics.series[0].tags).to.include('waf_version:0.0.1') expect(metrics.series[0].tags).to.include('event_rules_version:0.0.2') + expect(metrics.series[0].tags).to.include('success:true') + }) + + it('should increment waf.init and waf.config_errors on failed init', () => { + sinon.restore() + + appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion, false) + + const { metrics } = appsecNamespace.toJSON() + expect(metrics.series.length).to.be.eq(2) + expect(metrics.series[0].metric).to.be.eq('waf.init') + expect(metrics.series[0].tags).to.include('waf_version:0.0.1') + expect(metrics.series[0].tags).to.include('event_rules_version:0.0.2') + expect(metrics.series[0].tags).to.include('success:false') + + expect(metrics.series[1].metric).to.be.eq('waf.config_errors') + expect(metrics.series[1].tags).to.include('waf_version:0.0.1') + expect(metrics.series[1].tags).to.include('event_rules_version:0.0.2') }) }) describe('incWafUpdatesMetric', () => { it('should increment waf.updates metric', () => { - appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion) + appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion, true) expect(count).to.have.been.calledOnceWithExactly('waf.updates', { waf_version: wafVersion, - event_rules_version: rulesVersion + event_rules_version: rulesVersion, + success: true }) expect(inc).to.have.been.calledOnce }) @@ -225,9 +245,9 @@ describe('Appsec Waf Telemetry metrics', () => { it('should increment waf.updates metric multiple times', () => { sinon.restore() - appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion) - appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion) - appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion) + appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion, true) + appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion, true) + appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion, true) const { metrics } = appsecNamespace.toJSON() expect(metrics.series.length).to.be.eq(1) @@ -236,6 +256,24 @@ describe('Appsec Waf Telemetry metrics', () => { expect(metrics.series[0].points[0][1]).to.be.eq(3) expect(metrics.series[0].tags).to.include('waf_version:0.0.1') expect(metrics.series[0].tags).to.include('event_rules_version:0.0.2') + expect(metrics.series[0].tags).to.include('success:true') + }) + + it('should increment waf.updates and waf.config_errors on failed update', () => { + sinon.restore() + + appsecTelemetry.incrementWafUpdatesMetric(wafVersion, rulesVersion, false) + + const { metrics } = appsecNamespace.toJSON() + expect(metrics.series.length).to.be.eq(2) + expect(metrics.series[0].metric).to.be.eq('waf.updates') + expect(metrics.series[0].tags).to.include('waf_version:0.0.1') + expect(metrics.series[0].tags).to.include('event_rules_version:0.0.2') + expect(metrics.series[0].tags).to.include('success:false') + + expect(metrics.series[1].metric).to.be.eq('waf.config_errors') + expect(metrics.series[1].tags).to.include('waf_version:0.0.1') + expect(metrics.series[1].tags).to.include('event_rules_version:0.0.2') }) }) @@ -333,7 +371,7 @@ describe('Appsec Waf Telemetry metrics', () => { metrics: true }) - appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion) + appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion, true) expect(count).to.not.have.been.called expect(inc).to.not.have.been.called @@ -345,7 +383,7 @@ describe('Appsec Waf Telemetry metrics', () => { metrics: false }) - appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion) + appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion, true) expect(count).to.not.have.been.called expect(inc).to.not.have.been.called diff --git a/packages/dd-trace/test/appsec/waf/index.spec.js b/packages/dd-trace/test/appsec/waf/index.spec.js index 33c0bfbb3a3..6cd03abc005 100644 --- a/packages/dd-trace/test/appsec/waf/index.spec.js +++ b/packages/dd-trace/test/appsec/waf/index.spec.js @@ -24,7 +24,7 @@ describe('WAF Manager', () => { config = new Config() DDWAF = sinon.stub() - DDWAF.prototype.constructor.version = sinon.stub() + DDWAF.version = sinon.stub().returns('1.2.3') DDWAF.prototype.dispose = sinon.stub() DDWAF.prototype.createContext = sinon.stub() DDWAF.prototype.update = sinon.stub().callsFake(function (newRules) { @@ -53,6 +53,7 @@ describe('WAF Manager', () => { sinon.stub(Reporter, 'reportAttack') sinon.stub(Reporter, 'reportWafUpdate') sinon.stub(Reporter, 'reportDerivatives') + sinon.spy(Reporter, 'reportWafInit') webContext = {} sinon.stub(web, 'getContext').returns(webContext) @@ -70,21 +71,39 @@ describe('WAF Manager', () => { expect(waf.wafManager).not.to.be.null expect(waf.wafManager.ddwaf).to.be.instanceof(DDWAF) + expect(Reporter.reportWafInit).to.have.been.calledWithExactly( + '1.2.3', + '1.0.0', + { loaded: ['rule_1'], failed: [] }, + true + ) }) - it('should set init metrics without error', () => { - DDWAF.prototype.constructor.version.returns('1.2.3') + it('should handle failed DDWAF loading', () => { + const error = new Error('Failed to initialize DDWAF') + DDWAF.version.returns('1.2.3') + DDWAF.throws(error) + + try { + waf.init(rules, config.appsec) + expect.fail('waf init should have thrown an error') + } catch (err) { + expect(err).to.equal(error) + expect(Reporter.reportWafInit).to.have.been.calledWith('1.2.3', 'unknown') + } + }) + it('should set init metrics without error', () => { waf.init(rules, config.appsec) - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.waf.version', '1.2.3') - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.loaded', 1) - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.error_count', 0) - expect(Reporter.metricsQueue.set).not.to.been.calledWith('_dd.appsec.event_rules.errors') + expect(Reporter.metricsQueue.set).to.have.been.calledWithExactly('_dd.appsec.waf.version', '1.2.3') + expect(Reporter.metricsQueue.set).to.have.been.calledWithExactly('_dd.appsec.event_rules.loaded', 1) + expect(Reporter.metricsQueue.set).to.have.been.calledWithExactly('_dd.appsec.event_rules.error_count', 0) + expect(Reporter.metricsQueue.set).not.to.have.been.calledWith('_dd.appsec.event_rules.errors') }) it('should set init metrics with errors', () => { - DDWAF.prototype.constructor.version.returns('2.3.4') + DDWAF.version.returns('2.3.4') DDWAF.prototype.diagnostics = { rules: { loaded: ['rule_1'], @@ -98,10 +117,10 @@ describe('WAF Manager', () => { waf.init(rules, config.appsec) - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.waf.version', '2.3.4') - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.loaded', 1) - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.error_count', 2) - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.errors', + expect(Reporter.metricsQueue.set).to.have.been.calledWithExactly('_dd.appsec.waf.version', '2.3.4') + expect(Reporter.metricsQueue.set).to.have.been.calledWithExactly('_dd.appsec.event_rules.loaded', 1) + expect(Reporter.metricsQueue.set).to.have.been.calledWithExactly('_dd.appsec.event_rules.error_count', 2) + expect(Reporter.metricsQueue.set).to.have.been.calledWithExactly('_dd.appsec.event_rules.errors', '{"error_1":["invalid_1"],"error_2":["invalid_2","invalid_3"]}') }) }) @@ -134,7 +153,7 @@ describe('WAF Manager', () => { describe('wafManager.createDDWAFContext', () => { beforeEach(() => { - DDWAF.prototype.constructor.version.returns('4.5.6') + DDWAF.version.returns('4.5.6') waf.init(rules, config.appsec) }) @@ -155,7 +174,7 @@ describe('WAF Manager', () => { const wafVersion = '2.3.4' beforeEach(() => { - DDWAF.prototype.constructor.version.returns(wafVersion) + DDWAF.version.returns(wafVersion) waf.init(rules, config.appsec) }) @@ -179,10 +198,10 @@ describe('WAF Manager', () => { waf.update(rules) expect(DDWAF.prototype.update).to.be.calledOnceWithExactly(rules) - expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, '1.0.0') + expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, '1.0.0', true) }) - it('should call Reporter.reportWafUpdate', () => { + it('should call Reporter.reportWafUpdate on successful update', () => { const rules = { metadata: { rules_version: '4.2.0' @@ -203,7 +222,26 @@ describe('WAF Manager', () => { waf.update(rules) - expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, '4.2.0') + expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, '4.2.0', true) + }) + + it('should call Reporter.reportWafUpdate on failed update', () => { + const rules = { + metadata: { + rules_version: '4.2.0' + } + } + const error = new Error('Failed to update rules') + + DDWAF.prototype.update = sinon.stub().throws(error) + + try { + waf.update(rules) + expect.fail('waf.update should have thrown an error') + } catch (err) { + expect(err).to.equal(error) + expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, 'unknown', false) + } }) }) diff --git a/packages/dd-trace/test/appsec/waf/waf_manager.spec.js b/packages/dd-trace/test/appsec/waf/waf_manager.spec.js index ebb7d371049..9093f43d921 100644 --- a/packages/dd-trace/test/appsec/waf/waf_manager.spec.js +++ b/packages/dd-trace/test/appsec/waf/waf_manager.spec.js @@ -8,7 +8,7 @@ describe('WAFManager', () => { beforeEach(() => { DDWAF = sinon.stub() - DDWAF.prototype.constructor.version = sinon.stub() + DDWAF.version = sinon.stub() DDWAF.prototype.knownAddresses = knownAddresses DDWAF.prototype.diagnostics = {} DDWAF.prototype.createContext = sinon.stub() From 0c8751c44f61e272954e12c18212754ab7d9d882 Mon Sep 17 00:00:00 2001 From: Ilyas Shabi Date: Wed, 19 Mar 2025 15:48:25 +0100 Subject: [PATCH 03/32] bump native appsec package (#5439) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 8fbf26b9aea..8282bf8519e 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ }, "dependencies": { "@datadog/libdatadog": "^0.5.0", - "@datadog/native-appsec": "8.5.0", + "@datadog/native-appsec": "8.5.1", "@datadog/native-iast-rewriter": "2.8.0", "@datadog/native-iast-taint-tracking": "3.3.0", "@datadog/native-metrics": "^3.1.0", diff --git a/yarn.lock b/yarn.lock index 32531ab4f03..3a3df154e05 100644 --- a/yarn.lock +++ b/yarn.lock @@ -361,10 +361,10 @@ resolved "https://registry.yarnpkg.com/@datadog/libdatadog/-/libdatadog-0.5.0.tgz#0ef2a2a76bb9505a0e7e5bc9be1415b467dbf368" integrity sha512-YvLUVOhYVjJssm0f22/RnDQMc7ZZt/w1bA0nty1vvjyaDz5EWaHfWaaV4GYpCt5MRvnGjCBxIwwbRivmGseKeQ== -"@datadog/native-appsec@8.5.0": - version "8.5.0" - resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.5.0.tgz#cf4eea74a07085a0dc9f3e98c130736b38cd61c9" - integrity sha512-95y+fm7jd+3iknzuu57pWEPw9fcK9uSBCPiB4kSPHszHu3bESlZM553tc4ANsz+X3gMkYGVg2pgSydG77nSDJw== +"@datadog/native-appsec@8.5.1": + version "8.5.1" + resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.5.1.tgz#000fb06b91949d74298288ace5da51873922cc03" + integrity sha512-g6cjIafeObxVV+zJ2U1TDWVBio+MC8/4QR0EmgZ9afvhgtXRXyth3/DUOBSLUoMvCbduHwl6CV9sBf+tbSksVg== dependencies: node-gyp-build "^3.9.0" From e374119eacaf5f46cc08e3d6b5823b8add72faa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 19 Mar 2025 18:50:41 +0100 Subject: [PATCH 04/32] [test optimization] Improve playwright flakiness (this time for real?) (#5440) --- .github/workflows/project.yml | 23 +++++++++++++++- .../passing-test.js | 11 ++++++++ .../playwright/playwright.spec.js | 26 ++++++++++++++----- 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 integration-tests/ci-visibility/playwright-tests-test-capabilities/passing-test.js diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index fae9a9bfecb..0d464068f8e 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -61,11 +61,32 @@ jobs: env: DD_INJECTION_ENABLED: 'true' + integration-playwright: + strategy: + matrix: + version: [18, latest] + runs-on: ubuntu-latest + env: + DD_SERVICE: dd-trace-js-integration-tests + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 + DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 + with: + node-version: ${{ matrix.version }} + - uses: ./.github/actions/install + # Install system dependencies for playwright + - run: npx playwright install-deps + - run: yarn test:integration:playwright + env: + NODE_OPTIONS: '-r ./ci/init' + integration-ci: strategy: matrix: version: [18, latest] - framework: [cucumber, playwright, selenium, jest, mocha] + framework: [cucumber, selenium, jest, mocha] runs-on: ubuntu-latest env: DD_SERVICE: dd-trace-js-integration-tests diff --git a/integration-tests/ci-visibility/playwright-tests-test-capabilities/passing-test.js b/integration-tests/ci-visibility/playwright-tests-test-capabilities/passing-test.js new file mode 100644 index 00000000000..736db3aeb1c --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-test-capabilities/passing-test.js @@ -0,0 +1,11 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test('should work with passing tests', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello World' + ]) +}) diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 69bd83ca123..0ed40cdfe7d 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -47,15 +47,14 @@ versions.forEach((version) => { let sandbox, cwd, receiver, childProcess, webAppPort before(async function () { - // bump from 60 to 90 seconds because we also need to install dependencies and download chromium + // bump from 60 to 90 seconds because playwright is heavy this.timeout(90000) sandbox = await createSandbox([`@playwright/test@${version}`, 'typescript'], true) cwd = sandbox.folder const { NODE_OPTIONS, ...restOfEnv } = process.env - // Install system dependencies - execSync('npx playwright install-deps', { cwd, env: restOfEnv }) // Install chromium (configured in integration-tests/playwright.config.js) - execSync('npx playwright install', { cwd, env: restOfEnv }) + // *Be advised*: this means that we'll only be using chromium for this test suite + execSync('npx playwright install chromium', { cwd, env: restOfEnv, stdio: 'inherit' }) webAppPort = await getPort() webAppServer.listen(webAppPort) }) @@ -202,7 +201,9 @@ versions.forEach((version) => { }) }) - it('works when tests are compiled to a different location', (done) => { + it('works when tests are compiled to a different location', function (done) { + // this has shown some flakiness + this.retries(1) let testOutput = '' receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { @@ -215,7 +216,7 @@ versions.forEach((version) => { assert.include(testOutput, '1 passed') assert.include(testOutput, '1 skipped') assert.notInclude(testOutput, 'TypeError') - }).then(() => done()).catch(done) + }, 25000).then(() => done()).catch(done) childProcess = exec( 'node ./node_modules/typescript/bin/tsc' + @@ -1082,6 +1083,15 @@ versions.forEach((version) => { context('libraries capabilities', () => { it('adds capabilities to tests', (done) => { + receiver.setKnownTests( + { + playwright: { + 'passing-test.js': [ + 'should work with passing tests' + ] + } + } + ) receiver.setSettings({ flaky_test_retries_enabled: true, itr_enabled: false, @@ -1111,12 +1121,14 @@ versions.forEach((version) => { env: { ...getCiVisAgentlessConfig(receiver.port), PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-tests-test-capabilities', DD_TEST_SESSION_NAME: 'my-test-session-name' }, stdio: 'pipe' } ) - childProcess.on('exit', (exitCode) => { + + childProcess.on('exit', () => { eventsPromise.then(() => { done() }).catch(done) From 24a26071733fec6e8950cd7b6763c515466d0dcf Mon Sep 17 00:00:00 2001 From: Thomas Hunter II Date: Wed, 19 Mar 2025 12:12:27 -0700 Subject: [PATCH 05/32] dbm: tedious (sql server) service mode (#5375) --- .../datadog-instrumentations/src/tedious.js | 23 +++--- packages/datadog-plugin-tedious/src/index.js | 23 +++--- .../datadog-plugin-tedious/test/index.spec.js | 73 +++++++++++++++++-- packages/dd-trace/src/plugins/database.js | 8 +- 4 files changed, 96 insertions(+), 31 deletions(-) diff --git a/packages/datadog-instrumentations/src/tedious.js b/packages/datadog-instrumentations/src/tedious.js index 3a80b03ad75..36a21b3c233 100644 --- a/packages/datadog-instrumentations/src/tedious.js +++ b/packages/datadog-instrumentations/src/tedious.js @@ -16,7 +16,7 @@ addHook({ name: 'tedious', versions: ['>=1.0.0'] }, tedious => { return makeRequest.apply(this, arguments) } - const queryOrProcedure = getQueryOrProcedure(request) + const [queryOrProcedure, queryParent, queryField] = getQueryOrProcedure(request) if (!queryOrProcedure) { return makeRequest.apply(this, arguments) @@ -28,7 +28,9 @@ addHook({ name: 'tedious', versions: ['>=1.0.0'] }, tedious => { const connectionConfig = this.config return asyncResource.runInAsyncScope(() => { - startCh.publish({ queryOrProcedure, connectionConfig }) + const payload = { queryOrProcedure, connectionConfig } + startCh.publish(payload) + queryParent[queryField] = payload.sql const cb = callbackResource.bind(request.callback, request) request.callback = asyncResource.bind(function (error) { @@ -53,14 +55,15 @@ addHook({ name: 'tedious', versions: ['>=1.0.0'] }, tedious => { return tedious }) +// returns [queryOrProcedure, parentObjectToSet, propertyNameToSet] function getQueryOrProcedure (request) { - if (!request.parameters) return - - const statement = request.parametersByName.statement || request.parametersByName.stmt - - if (!statement) { - return request.sqlTextOrProcedure + if (!request.parameters) return [null] + + if (request.parametersByName.statement) { + return [request.parametersByName.statement.value, request.parametersByName.statement, 'value'] + } else if (request.parametersByName.stmt) { + return [request.parametersByName.stmt.value, request.parametersByName.stmt, 'value'] + } else { + return [request.sqlTextOrProcedure, request, 'sqlTextOrProcedure'] } - - return statement.value } diff --git a/packages/datadog-plugin-tedious/src/index.js b/packages/datadog-plugin-tedious/src/index.js index 97eab3617dc..7fba730c9d2 100644 --- a/packages/datadog-plugin-tedious/src/index.js +++ b/packages/datadog-plugin-tedious/src/index.js @@ -8,22 +8,27 @@ class TediousPlugin extends DatabasePlugin { static get operation () { return 'request' } // TODO: change to match other database plugins static get system () { return 'mssql' } - start ({ queryOrProcedure, connectionConfig }) { - this.startSpan(this.operationName(), { - service: this.serviceName({ pluginConfig: this.config, system: this.system }), - resource: queryOrProcedure, + start (payload) { + const service = this.serviceName({ pluginConfig: this.config, system: this.system }) + const span = this.startSpan(this.operationName(), { + service, + resource: payload.queryOrProcedure, type: 'sql', kind: 'client', meta: { 'db.type': 'mssql', component: 'tedious', - 'out.host': connectionConfig.server, - [CLIENT_PORT_KEY]: connectionConfig.options.port, - 'db.user': connectionConfig.userName || connectionConfig.authentication.options.userName, - 'db.name': connectionConfig.options.database, - 'db.instance': connectionConfig.options.instanceName + 'out.host': payload.connectionConfig.server, + [CLIENT_PORT_KEY]: payload.connectionConfig.options.port, + 'db.user': payload.connectionConfig.userName || payload.connectionConfig.authentication.options.userName, + 'db.name': payload.connectionConfig.options.database, + 'db.instance': payload.connectionConfig.options.instanceName } }) + + // SQL Server includes comments when caching queries + // For that reason we allow service mode but not full mode + payload.sql = this.injectDbmQuery(span, payload.queryOrProcedure, service, true) } } diff --git a/packages/datadog-plugin-tedious/test/index.spec.js b/packages/datadog-plugin-tedious/test/index.spec.js index e3fc9479b71..702714b2c20 100644 --- a/packages/datadog-plugin-tedious/test/index.spec.js +++ b/packages/datadog-plugin-tedious/test/index.spec.js @@ -12,7 +12,6 @@ describe('Plugin', () => { let tds let tracer let connection - let connectionIsClosed withVersions('tedious', 'tedious', version => { beforeEach(() => { @@ -53,7 +52,6 @@ describe('Plugin', () => { config.password = MSSQL_PASSWORD } - connectionIsClosed = false connection = new tds.Connection(config) .on('connect', done) @@ -64,12 +62,8 @@ describe('Plugin', () => { }) afterEach(function (done) { - if (connectionIsClosed) { - done() - } else { - connection.on('end', () => done()) - connection.close() - } + connection.on('end', () => done()) + connection.close() }) withNamingSchema( @@ -405,5 +399,68 @@ describe('Plugin', () => { }) } }) + + // it's a pretty old version with a different enough API that I don't think it's worth supporting + const testDbm = semver.intersects(version, '<10') ? describe.skip : describe + testDbm('with configuration and DBM enabled', () => { + let config + let tds + let connection + + beforeEach(() => { + return agent.load('tedious', { dbmPropagationMode: 'service', service: 'custom' }).then(() => { + tds = require(`../../../versions/tedious@${version}`).get() + }) + }) + + afterEach(() => { + return agent.close({ ritmReset: false }) + }) + + beforeEach((done) => { + config = { + server: 'localhost', + options: { + database: 'master', + trustServerCertificate: true + }, + authentication: { + options: { + userName: MSSQL_USERNAME, + password: MSSQL_PASSWORD + }, + type: 'default' + } + } + + connection = new tds.Connection(config) + .on('connect', done) + + connection.connect() + }) + + afterEach(function (done) { + connection.on('end', () => done()) + connection.close() + }) + + it('should inject the correct DBM comment into query but not into trace', done => { + const query = 'SELECT 1 + 1 AS solution' + + const request = new tds.Request(query, (err) => { + if (err) return done(err) + promise.then(done, done) + }) + + const promise = agent + .use(traces => { + expect(traces[0][0]).to.have.property('resource', 'SELECT 1 + 1 AS solution') + expect(request.sqlTextOrProcedure).to.equal("/*dddb='master',dddbs='custom',dde='tester'," + + "ddh='localhost',ddps='test',ddpv='10.8.2'*/ SELECT 1 + 1 AS solution") + }) + + connection.execSql(request) + }) + }) }) }) diff --git a/packages/dd-trace/src/plugins/database.js b/packages/dd-trace/src/plugins/database.js index cd688133761..dc0c53c982d 100644 --- a/packages/dd-trace/src/plugins/database.js +++ b/packages/dd-trace/src/plugins/database.js @@ -63,7 +63,7 @@ class DatabasePlugin extends StoragePlugin { return tracerService } - createDbmComment (span, serviceName, isPreparedStatement = false) { + createDbmComment (span, serviceName, disableFullMode = false) { const mode = this.config.dbmPropagationMode const dbmService = this.getDbmServiceName(span, serviceName) @@ -73,7 +73,7 @@ class DatabasePlugin extends StoragePlugin { const servicePropagation = this.createDBMPropagationCommentService(dbmService, span) - if (isPreparedStatement || mode === 'service') { + if (disableFullMode || mode === 'service') { return servicePropagation } else if (mode === 'full') { span.setTag('_dd.dbm_trace_injected', 'true') @@ -82,8 +82,8 @@ class DatabasePlugin extends StoragePlugin { } } - injectDbmQuery (span, query, serviceName, isPreparedStatement = false) { - const dbmTraceComment = this.createDbmComment(span, serviceName, isPreparedStatement) + injectDbmQuery (span, query, serviceName, disableFullMode = false) { + const dbmTraceComment = this.createDbmComment(span, serviceName, disableFullMode) if (!dbmTraceComment) { return query From edc0de46d966af9619ea5754bbc29a931b569329 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Wed, 19 Mar 2025 18:18:11 -0400 Subject: [PATCH 06/32] increase next.js test timeout to 5 minutes (#5442) --- packages/datadog-plugin-next/test/index.spec.js | 4 ++-- .../datadog-plugin-next/test/integration-test/client.spec.js | 4 ++-- packages/dd-trace/test/appsec/index.next.plugin.spec.js | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/datadog-plugin-next/test/index.spec.js b/packages/datadog-plugin-next/test/index.spec.js index 3fa35e4e280..248b4247fe2 100644 --- a/packages/datadog-plugin-next/test/index.spec.js +++ b/packages/datadog-plugin-next/test/index.spec.js @@ -30,7 +30,7 @@ describe('Plugin', function () { }) before(function (done) { - this.timeout(120000) + this.timeout(300 * 1000) const cwd = standalone ? path.join(__dirname, '.next/standalone') : __dirname @@ -68,7 +68,7 @@ describe('Plugin', function () { }) after(async function () { - this.timeout(5000) + this.timeout(30 * 1000) server.kill() diff --git a/packages/datadog-plugin-next/test/integration-test/client.spec.js b/packages/datadog-plugin-next/test/integration-test/client.spec.js index 841e9402584..76b114bc76c 100644 --- a/packages/datadog-plugin-next/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-next/test/integration-test/client.spec.js @@ -19,7 +19,7 @@ describe('esm', () => { withVersions('next', 'next', '>=11.1', version => { before(async function () { // next builds slower in the CI, match timeout with unit tests - this.timeout(120 * 1000) + this.timeout(300 * 1000) sandbox = await createSandbox([`'next@${version}'`, 'react@^18.2.0', 'react-dom@^18.2.0'], false, ['./packages/datadog-plugin-next/test/integration-test/*'], 'NODE_OPTIONS=--openssl-legacy-provider yarn exec next build') @@ -47,6 +47,6 @@ describe('esm', () => { assert.isArray(payload) assert.strictEqual(checkSpansForServiceName(payload, 'next.request'), true) }, undefined, undefined, true) - }).timeout(120 * 1000) + }).timeout(300 * 1000) }) }) diff --git a/packages/dd-trace/test/appsec/index.next.plugin.spec.js b/packages/dd-trace/test/appsec/index.next.plugin.spec.js index de711c5ff94..295ac76f3bd 100644 --- a/packages/dd-trace/test/appsec/index.next.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.next.plugin.spec.js @@ -23,7 +23,7 @@ describe('test suite', () => { const appDir = path.join(__dirname, 'next', appName) before(async function () { - this.timeout(120 * 1000) // Webpack is very slow and builds on every test run + this.timeout(300 * 1000) // Webpack is very slow and builds on every test run const cwd = appDir From 00b1529476561d2c6c3edf53df50fd36d07442aa Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Wed, 19 Mar 2025 19:52:59 -0400 Subject: [PATCH 07/32] lazy load dd-trace-api integration (#5406) --- packages/datadog-instrumentations/src/dd-trace-api.js | 7 +++++++ packages/datadog-instrumentations/src/helpers/hooks.js | 1 + packages/datadog-instrumentations/src/helpers/register.js | 2 +- packages/datadog-plugin-dd-trace-api/test/index.spec.js | 3 +++ packages/dd-trace/src/plugin_manager.js | 3 --- 5 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 packages/datadog-instrumentations/src/dd-trace-api.js diff --git a/packages/datadog-instrumentations/src/dd-trace-api.js b/packages/datadog-instrumentations/src/dd-trace-api.js new file mode 100644 index 00000000000..bcf14763f0f --- /dev/null +++ b/packages/datadog-instrumentations/src/dd-trace-api.js @@ -0,0 +1,7 @@ +'use strict' + +const { addHook } = require('./helpers/instrument') + +// Empty hook just to make the plugin load. +// TODO: Add version range when the module is released on npm. +addHook({ name: 'dd-trace-api' }, api => api) diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index f60fd297f75..3cc455fc120 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -43,6 +43,7 @@ module.exports = { couchbase: () => require('../couchbase'), crypto: () => require('../crypto'), cypress: () => require('../cypress'), + 'dd-trace-api': () => require('../dd-trace-api'), dns: () => require('../dns'), elasticsearch: () => require('../elasticsearch'), express: () => require('../express'), diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index 8c07683623d..78f0f483eac 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -158,7 +158,7 @@ for (const packageName of names) { } function matchVersion (version, ranges) { - return !version || (ranges && ranges.some(range => satisfies(version, range))) + return !version || !ranges || ranges.some(range => satisfies(version, range)) } function getVersion (moduleBaseDir) { diff --git a/packages/datadog-plugin-dd-trace-api/test/index.spec.js b/packages/datadog-plugin-dd-trace-api/test/index.spec.js index b02109c4aee..ae006c3fc00 100644 --- a/packages/datadog-plugin-dd-trace-api/test/index.spec.js +++ b/packages/datadog-plugin-dd-trace-api/test/index.spec.js @@ -22,6 +22,9 @@ describe('Plugin', () => { tracer = require('../../dd-trace') + // TODO: Use the real module when it's released. + dc.channel('dd-trace:instrumentation:load').publish({ name: 'dd-trace-api' }) + sinon.spy(tracer) sinon.spy(tracer.appsec) sinon.spy(tracer.dogstatsd) diff --git a/packages/dd-trace/src/plugin_manager.js b/packages/dd-trace/src/plugin_manager.js index 96b82a7f9de..a3948a923e5 100644 --- a/packages/dd-trace/src/plugin_manager.js +++ b/packages/dd-trace/src/plugin_manager.js @@ -28,9 +28,6 @@ loadChannel.subscribe(({ name }) => { maybeEnable(plugins[name]) }) -// Always enabled -maybeEnable(require('../../datadog-plugin-dd-trace-api/src')) - function maybeEnable (Plugin) { if (!Plugin || typeof Plugin !== 'function') return if (!pluginClasses[Plugin.id]) { From 9f46dbc5ce8a2e79523b4661500704bcbb8a2c2b Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Wed, 19 Mar 2025 22:12:22 -0400 Subject: [PATCH 08/32] retry test agent start when it fails (#5443) --- .github/actions/testagent/start/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/testagent/start/action.yml b/.github/actions/testagent/start/action.yml index d17071f6680..60734e09609 100644 --- a/.github/actions/testagent/start/action.yml +++ b/.github/actions/testagent/start/action.yml @@ -4,5 +4,5 @@ runs: using: composite steps: - uses: actions/checkout@v4 - - run: docker compose up -d testagent + - run: docker compose up -d testagent || docker compose up -d testagent shell: bash From aca4f0564fb38df827c9d8441466f4933d5cae0b Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 20 Mar 2025 07:03:24 -0400 Subject: [PATCH 09/32] fix mysql/mysql2/pg dbm trace id tests (#5436) * fix mysql dbm trace id test * fix mysql2 and pg * code cleanup --- packages/datadog-plugin-mysql/test/index.spec.js | 8 ++------ packages/datadog-plugin-mysql2/test/index.spec.js | 8 ++------ packages/datadog-plugin-pg/test/index.spec.js | 8 ++------ 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/packages/datadog-plugin-mysql/test/index.spec.js b/packages/datadog-plugin-mysql/test/index.spec.js index d18bd302aa8..217de59f6ee 100644 --- a/packages/datadog-plugin-mysql/test/index.spec.js +++ b/packages/datadog-plugin-mysql/test/index.spec.js @@ -460,7 +460,7 @@ describe('Plugin', () => { it('query text should contain traceparent', done => { let queryText = '' agent.use(traces => { - const expectedTimePrefix = Math.floor(clock.now / 1000).toString(16).padStart(8, '0').padEnd(16, '0') + const expectedTimePrefix = traces[0][0].meta['_dd.p.tid'].toString(16).padStart(16, '0') const traceId = expectedTimePrefix + traces[0][0].trace_id.toString(16).padStart(16, '0') const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') @@ -468,9 +468,7 @@ describe('Plugin', () => { `/*dddb='db',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT 1 + 1 AS solution`) }).then(done, done) - const clock = sinon.useFakeTimers(new Date()) connection.query('SELECT 1 + 1 AS solution', () => { - clock.restore() queryText = connection._protocol._queue[0].sql }) }) @@ -542,7 +540,7 @@ describe('Plugin', () => { it('query text should contain traceparent', done => { let queryText = '' agent.use(traces => { - const expectedTimePrefix = Math.floor(clock.now / 1000).toString(16).padStart(8, '0').padEnd(16, '0') + const expectedTimePrefix = traces[0][0].meta['_dd.p.tid'].toString(16).padStart(16, '0') const traceId = expectedTimePrefix + traces[0][0].trace_id.toString(16).padStart(16, '0') const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') @@ -550,9 +548,7 @@ describe('Plugin', () => { `/*dddb='db',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT 1 + 1 AS solution`) }).then(done, done) - const clock = sinon.useFakeTimers(new Date()) pool.query('SELECT 1 + 1 AS solution', () => { - clock.restore() queryText = pool._allConnections[0]._protocol._queue[0].sql }) }) diff --git a/packages/datadog-plugin-mysql2/test/index.spec.js b/packages/datadog-plugin-mysql2/test/index.spec.js index f6e7cde5a03..6ce8b2a5400 100644 --- a/packages/datadog-plugin-mysql2/test/index.spec.js +++ b/packages/datadog-plugin-mysql2/test/index.spec.js @@ -447,7 +447,7 @@ describe('Plugin', () => { it('query text should contain traceparent', done => { let queryText = '' agent.use(traces => { - const expectedTimePrefix = Math.floor(clock.now / 1000).toString(16).padStart(8, '0').padEnd(16, '0') + const expectedTimePrefix = traces[0][0].meta['_dd.p.tid'].toString(16).padStart(16, '0') const traceId = expectedTimePrefix + traces[0][0].trace_id.toString(16).padStart(16, '0') const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') @@ -455,9 +455,7 @@ describe('Plugin', () => { `/*dddb='db',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT 1 + 1 AS solution`) }).then(done, done) - const clock = sinon.useFakeTimers(new Date()) const connect = connection.query('SELECT 1 + 1 AS solution', () => { - clock.restore() queryText = connect.sql }) }) @@ -529,7 +527,7 @@ describe('Plugin', () => { it('query text should contain traceparent', done => { let queryText = '' agent.use(traces => { - const expectedTimePrefix = Math.floor(clock.now / 1000).toString(16).padStart(8, '0').padEnd(16, '0') + const expectedTimePrefix = traces[0][0].meta['_dd.p.tid'].toString(16).padStart(16, '0') const traceId = expectedTimePrefix + traces[0][0].trace_id.toString(16).padStart(16, '0') const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') @@ -537,9 +535,7 @@ describe('Plugin', () => { `/*dddb='db',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT 1 + 1 AS solution`) }).then(done, done) - const clock = sinon.useFakeTimers(new Date()) const queryPool = pool.query('SELECT 1 + 1 AS solution', () => { - clock.restore() queryText = queryPool.sql }) }) diff --git a/packages/datadog-plugin-pg/test/index.spec.js b/packages/datadog-plugin-pg/test/index.spec.js index 72dab8b3ece..6d80edb2ec4 100644 --- a/packages/datadog-plugin-pg/test/index.spec.js +++ b/packages/datadog-plugin-pg/test/index.spec.js @@ -486,16 +486,14 @@ describe('Plugin', () => { it('query text should contain traceparent', done => { agent.use(traces => { - const expectedTimePrefix = Math.floor(clock.now / 1000).toString(16).padStart(8, '0').padEnd(16, '0') + const expectedTimePrefix = traces[0][0].meta['_dd.p.tid'].toString(16).padStart(16, '0') const traceId = expectedTimePrefix + traces[0][0].trace_id.toString(16).padStart(16, '0') const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') expect(seenTraceId).to.equal(traceId) expect(seenSpanId).to.equal(spanId) }).then(done, done) - const clock = sinon.useFakeTimers(new Date()) client.query('SELECT $1::text as message', ['Hello World!'], (err, result) => { - clock.restore() if (err) return done(err) expect(seenTraceParent).to.be.true client.end((err) => { @@ -568,7 +566,7 @@ describe('Plugin', () => { } agent.use(traces => { - const expectedTimePrefix = Math.floor(clock.now / 1000).toString(16).padStart(8, '0').padEnd(16, '0') + const expectedTimePrefix = traces[0][0].meta['_dd.p.tid'].toString(16).padStart(16, '0') const traceId = expectedTimePrefix + traces[0][0].trace_id.toString(16).padStart(16, '0') const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') @@ -577,9 +575,7 @@ describe('Plugin', () => { `traceparent='00-${traceId}-${spanId}-00'*/ SELECT $1::text as message`) }).then(done, done) - const clock = sinon.useFakeTimers(new Date()) client.query(query, ['Hello world!'], (err) => { - clock.restore() if (err) return done(err) client.end((err) => { From d5a436a22c27a5e1dd84e9f212b43e8959f598b1 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 20 Mar 2025 10:37:19 -0400 Subject: [PATCH 10/32] fix amqplib flaky dsm tests (#5445) --- .../datadog-plugin-amqplib/test/index.spec.js | 120 ++++++++++-------- 1 file changed, 67 insertions(+), 53 deletions(-) diff --git a/packages/datadog-plugin-amqplib/test/index.spec.js b/packages/datadog-plugin-amqplib/test/index.spec.js index 70b9fa394d4..67ad8f63872 100644 --- a/packages/datadog-plugin-amqplib/test/index.spec.js +++ b/packages/datadog-plugin-amqplib/test/index.spec.js @@ -2,6 +2,9 @@ const agent = require('../../dd-trace/test/plugins/agent') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') +const id = require('../../dd-trace/src/id') +const { ENTRY_PARENT_HASH } = require('../../dd-trace/src/datastreams/processor') +const { computePathwayHash } = require('../../dd-trace/src/datastreams/pathway') const { expectedSchema, rawExpectedSchema } = require('./naming') @@ -9,12 +12,14 @@ describe('Plugin', () => { let tracer let connection let channel + let queue describe('amqplib', () => { withVersions('amqplib', 'amqplib', version => { beforeEach(() => { process.env.DD_DATA_STREAMS_ENABLED = 'true' tracer = require('../../dd-trace') + queue = `test-${id()}` }) afterEach(() => { @@ -40,26 +45,22 @@ describe('Plugin', () => { }) }) + afterEach(() => { + return agent.close({ ritmReset: false }) + }) + describe('without plugin', () => { it('should run commands normally', done => { - channel.assertQueue('test', {}, () => { done() }) + channel.assertQueue(queue, {}, () => { done() }) }) }) describe('when using a callback', () => { - before(() => { - return agent.load('amqplib') - }) - - after(() => { - return agent.close({ ritmReset: false }) - }) - describe('when sending commands', () => { withPeerService( () => tracer, 'amqplib', - () => channel.assertQueue('test', {}, () => {}), + () => channel.assertQueue(queue, {}, () => {}), 'localhost', 'out.host' ) @@ -70,7 +71,7 @@ describe('Plugin', () => { const span = traces[0][0] expect(span).to.have.property('name', expectedSchema.controlPlane.opName) expect(span).to.have.property('service', expectedSchema.controlPlane.serviceName) - expect(span).to.have.property('resource', 'queue.declare test') + expect(span).to.have.property('resource', `queue.declare ${queue}`) expect(span).to.not.have.property('type') expect(span.meta).to.have.property('span.kind', 'client') expect(span.meta).to.have.property('out.host', 'localhost') @@ -80,7 +81,7 @@ describe('Plugin', () => { .then(done) .catch(done) - channel.assertQueue('test', {}, () => {}) + channel.assertQueue(queue, {}, () => {}) }) it('should do automatic instrumentation for queued commands', done => { @@ -90,7 +91,7 @@ describe('Plugin', () => { expect(span).to.have.property('name', expectedSchema.controlPlane.opName) expect(span).to.have.property('service', expectedSchema.controlPlane.serviceName) - expect(span).to.have.property('resource', 'queue.delete test') + expect(span).to.have.property('resource', `queue.delete ${queue}`) expect(span).to.not.have.property('type') expect(span.meta).to.have.property('span.kind', 'client') expect(span.meta).to.have.property('out.host', 'localhost') @@ -100,8 +101,8 @@ describe('Plugin', () => { .then(done) .catch(done) - channel.assertQueue('test', {}, () => {}) - channel.deleteQueue('test', () => {}) + channel.assertQueue(queue, {}, () => {}) + channel.deleteQueue(queue, () => {}) }) it('should handle errors', done => { @@ -128,7 +129,7 @@ describe('Plugin', () => { }) withNamingSchema( - () => channel.assertQueue('test', {}, () => {}), + () => channel.assertQueue(queue, {}, () => {}), rawExpectedSchema.controlPlane ) }) @@ -137,7 +138,7 @@ describe('Plugin', () => { withPeerService( () => tracer, 'amqplib', - () => channel.assertQueue('test', {}, () => {}), + () => channel.assertQueue(queue, {}, () => {}), 'localhost', 'out.host' ) @@ -181,7 +182,7 @@ describe('Plugin', () => { .catch(done) try { - channel.sendToQueue('test', 'invalid') + channel.sendToQueue(queue, 'invalid') } catch (e) { error = e } @@ -293,7 +294,7 @@ describe('Plugin', () => { }) it('should run the callback in the parent context', done => { - channel.assertQueue('test', {}) + channel.assertQueue(queue, {}) .then(() => { expect(tracer.scope().active()).to.be.null done() @@ -305,22 +306,32 @@ describe('Plugin', () => { describe('when data streams monitoring is enabled', function () { this.timeout(10000) - const expectedProducerHashWithTopic = '16804605750389532869' - const expectedProducerHashWithExchange = '2722596631431228032' - - const expectedConsumerHash = '17529824252700998941' - - before(() => { - tracer = require('../../dd-trace') - tracer.use('amqplib') - }) + let expectedProducerHashWithTopic + let expectedProducerHashWithExchange + let expectedConsumerHash - before(async () => { - return agent.load('amqplib') - }) - - after(() => { - return agent.close({ ritmReset: false }) + beforeEach(() => { + const producerHashWithTopic = computePathwayHash('test', 'tester', [ + 'direction:out', + 'has_routing_key:true', + `topic:${queue}`, + 'type:rabbitmq' + ], ENTRY_PARENT_HASH) + + expectedProducerHashWithTopic = producerHashWithTopic.readBigUInt64BE(0).toString() + + expectedProducerHashWithExchange = computePathwayHash('test', 'tester', [ + 'direction:out', + 'exchange:namedExchange', + 'has_routing_key:true', + 'type:rabbitmq' + ], ENTRY_PARENT_HASH).readBigUInt64BE(0).toString() + + expectedConsumerHash = computePathwayHash('test', 'tester', [ + 'direction:in', + `topic:${queue}`, + 'type:rabbitmq' + ], producerHashWithTopic).readBigUInt64BE(0).toString() }) it('Should emit DSM stats to the agent when sending a message on an unnamed exchange', done => { @@ -338,13 +349,13 @@ describe('Plugin', () => { expect(statsPointsReceived[0].EdgeTags).to.deep.equal([ 'direction:out', 'has_routing_key:true', - 'topic:testDSM', + `topic:${queue}`, 'type:rabbitmq' ]) expect(agent.dsmStatsExist(agent, expectedProducerHashWithTopic)).to.equal(true) }, { timeoutMs: 10000 }).then(done, done) - channel.assertQueue('testDSM', {}, (err, ok) => { + channel.assertQueue(queue, {}, (err, ok) => { if (err) return done(err) channel.sendToQueue(ok.queue, Buffer.from('DSM pathway test')) @@ -390,15 +401,16 @@ describe('Plugin', () => { }) } }) - expect(statsPointsReceived.length).to.be.at.least(1) - expect(statsPointsReceived[0].EdgeTags).to.deep.equal( - ['direction:in', 'topic:testDSM', 'type:rabbitmq']) + expect(statsPointsReceived.length).to.equal(2) + expect(statsPointsReceived[1].EdgeTags).to.deep.equal( + ['direction:in', `topic:${queue}`, 'type:rabbitmq']) expect(agent.dsmStatsExist(agent, expectedConsumerHash)).to.equal(true) }, { timeoutMs: 10000 }).then(done, done) - channel.assertQueue('testDSM', {}, (err, ok) => { + channel.assertQueue(queue, {}, (err, ok) => { if (err) return done(err) + channel.sendToQueue(ok.queue, Buffer.from('DSM pathway test')) channel.consume(ok.queue, () => {}, {}, (err, ok) => { if (err) done(err) }) @@ -416,17 +428,17 @@ describe('Plugin', () => { }) } }) - expect(statsPointsReceived.length).to.be.at.least(1) + expect(statsPointsReceived.length).to.equal(1) expect(statsPointsReceived[0].EdgeTags).to.deep.equal([ 'direction:out', 'has_routing_key:true', - 'topic:testDSM', + `topic:${queue}`, 'type:rabbitmq' ]) expect(agent.dsmStatsExist(agent, expectedProducerHashWithTopic)).to.equal(true) }, { timeoutMs: 10000 }).then(done, done) - channel.assertQueue('testDSM', {}, (err, ok) => { + channel.assertQueue(queue, {}, (err, ok) => { if (err) return done(err) channel.sendToQueue(ok.queue, Buffer.from('DSM pathway test')) @@ -444,15 +456,16 @@ describe('Plugin', () => { }) } }) - expect(statsPointsReceived.length).to.be.at.least(1) - expect(statsPointsReceived[0].EdgeTags).to.deep.equal( - ['direction:in', 'topic:testDSM', 'type:rabbitmq']) + expect(statsPointsReceived.length).to.equal(2) + expect(statsPointsReceived[1].EdgeTags).to.deep.equal( + ['direction:in', `topic:${queue}`, 'type:rabbitmq']) expect(agent.dsmStatsExist(agent, expectedConsumerHash)).to.equal(true) }, { timeoutMs: 10000 }).then(done, done) - channel.assertQueue('testDSM', {}, (err, ok) => { + channel.assertQueue(queue, {}, (err, ok) => { if (err) return done(err) + channel.sendToQueue(ok.queue, Buffer.from('DSM pathway test')) channel.get(ok.queue, {}, (err, ok) => { if (err) done(err) }) @@ -460,7 +473,7 @@ describe('Plugin', () => { }) it('Should set pathway hash tag on a span when producing', (done) => { - channel.assertQueue('testDSM', {}, (err, ok) => { + channel.assertQueue(queue, {}, (err, ok) => { if (err) return done(err) channel.sendToQueue(ok.queue, Buffer.from('dsm test')) @@ -481,9 +494,10 @@ describe('Plugin', () => { }) it('Should set pathway hash tag on a span when consuming', (done) => { - channel.assertQueue('testDSM', {}, (err, ok) => { + channel.assertQueue(queue, {}, (err, ok) => { if (err) return done(err) + channel.sendToQueue(ok.queue, Buffer.from('dsm test')) channel.consume(ok.queue, () => {}, {}, (err, ok) => { if (err) return done(err) @@ -506,7 +520,7 @@ describe('Plugin', () => { }) describe('with configuration', () => { - after(() => { + afterEach(() => { return agent.close({ ritmReset: false }) }) @@ -531,16 +545,16 @@ describe('Plugin', () => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'test-custom-service') - expect(traces[0][0]).to.have.property('resource', 'queue.declare test') + expect(traces[0][0]).to.have.property('resource', `queue.declare ${queue}`) }, 2) .then(done) .catch(done) - channel.assertQueue('test', {}, () => {}) + channel.assertQueue(queue, {}, () => {}) }) withNamingSchema( - () => channel.assertQueue('test', {}, () => {}), + () => channel.assertQueue(queue, {}, () => {}), { v0: { opName: 'amqp.command', From 748339131501126add74f30eae1f50efa0b1d7d8 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 20 Mar 2025 10:38:27 -0400 Subject: [PATCH 11/32] fix aws-sdk kinesis flaky dsm test (#5446) --- packages/datadog-plugin-aws-sdk/test/kinesis.spec.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js index 2e3bf356f3e..4b4a6684eb8 100644 --- a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js @@ -354,9 +354,7 @@ describe('Kinesis', function () { }, { timeoutMs: 10000 }).then(done, done) helpers.putTestRecords(kinesis, streamNameDSM, (err, data) => { - if (err) return done(err) - - nowStub.restore() + // Swallow the error as it doesn't matter for this test. }) }) }) From a68c72ae3c871e874d9dedcc04b45dddc6addfb5 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 20 Mar 2025 12:36:07 -0400 Subject: [PATCH 12/32] fix profiling test expectating minimum 2 requests instead of 1 (#5449) --- integration-tests/profiler/profiler.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/profiler/profiler.spec.js b/integration-tests/profiler/profiler.spec.js index 6c7f4942e1e..0fc16da2475 100644 --- a/integration-tests/profiler/profiler.spec.js +++ b/integration-tests/profiler/profiler.spec.js @@ -692,9 +692,9 @@ describe('profiler', () => { assert.equal(series[0].metric, 'profile_api.requests') assert.equal(series[0].type, 'count') // There's a race between metrics and on-shutdown profile, so metric - // value will be between 2 and 3 + // value will be between 1 and 3 requestCount = series[0].points[0][1] - assert.isAtLeast(requestCount, 2) + assert.isAtLeast(requestCount, 1) assert.isAtMost(requestCount, 3) assert.equal(series[1].metric, 'profile_api.responses') From bcd9fbd522d9013e004c6353698e917d40917d47 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 20 Mar 2025 16:40:11 -0400 Subject: [PATCH 13/32] add matrix strategy to llmobs sdk ci job (#5451) --- .github/workflows/llmobs.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/llmobs.yml b/.github/workflows/llmobs.yml index 0d244ff98f9..0a689f5fc65 100644 --- a/.github/workflows/llmobs.yml +++ b/.github/workflows/llmobs.yml @@ -15,23 +15,22 @@ concurrency: jobs: sdk: + strategy: + matrix: + version: [18, 20, 22, latest] runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/oldest-maintenance-lts + - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 + with: + node-version: ${{ matrix.version }} - uses: ./.github/actions/install - run: yarn test:llmobs:sdk:ci - - uses: ./.github/actions/node/newest-maintenance-lts - - run: yarn test:llmobs:sdk:ci - - uses: ./.github/actions/node/active-lts - - run: yarn test:llmobs:sdk:ci - - uses: ./.github/actions/node/latest - - run: yarn test:llmobs:sdk:ci - if: always() uses: ./.github/actions/testagent/logs with: - suffix: llmobs-${{ github.job }} + suffix: llmobs-${{ github.job }}-${{ matrix.version }} - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 openai: From 5bdc34218985de58a876093cce2621edebfefa6d Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 20 Mar 2025 22:16:27 -0400 Subject: [PATCH 14/32] retry npm install/view for appsec and plugins tests (#5434) --- .github/workflows/appsec.yml | 2 +- scripts/install_plugin_modules.js | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index a8539184b67..54bb4da2ad5 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -257,7 +257,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest-maintenance-lts - run: yarn test:integration:appsec - uses: ./.github/actions/node/active-lts diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index 212dc5928ed..cfcc9b29371 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -161,12 +161,11 @@ async function getVersionList (name) { return list } -function npmView (input) { +function npmView (input, retry = true) { return new Promise((resolve, reject) => { childProcess.exec(`npm view ${input} --json`, (err, stdout) => { if (err) { - reject(err) - return + return retry ? npmView(input, false).then(resolve, reject) : reject(err) } resolve(JSON.parse(stdout.toString('utf8'))) }) From 0a33d5dcfbd0343e13ffb5572d3b2410e8c32c4b Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 20 Mar 2025 23:16:18 -0400 Subject: [PATCH 15/32] set dc-polyfill version to 0.1.6 (#5457) --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 8282bf8519e..c66216adaac 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "@opentelemetry/api": ">=1.0.0 <1.9.0", "@opentelemetry/core": "^1.14.0", "crypto-randomuuid": "^1.0.0", - "dc-polyfill": "^0.1.4", + "dc-polyfill": "0.1.6", "ignore": "^5.2.4", "import-in-the-middle": "1.13.1", "istanbul-lib-coverage": "3.2.0", diff --git a/yarn.lock b/yarn.lock index 3a3df154e05..c76f9fa8d99 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1788,7 +1788,7 @@ data-view-byte-offset@^1.0.1: es-errors "^1.3.0" is-data-view "^1.0.1" -dc-polyfill@^0.1.4: +dc-polyfill@0.1.6: version "0.1.6" resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.6.tgz#c2940fa68ffb24a7bf127cc6cfdd15b39f0e7f02" integrity sha512-UV33cugmCC49a5uWAApM+6Ev9ZdvIUMTrtCO9fj96TPGOQiea54oeO3tiEVdVeo3J9N2UdJEmbS4zOkkEA35uQ== From b18ea0ab5a8bba5399ddb703ac2b97a88f94f0e0 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Fri, 21 Mar 2025 10:36:17 -0400 Subject: [PATCH 16/32] replace node action cache with custom cache in ci (#5454) --- .github/actions/install/action.yml | 21 +++++++++++++++++-- .github/actions/node/active-lts/action.yml | 1 - .github/actions/node/latest/action.yml | 1 - .../node/newest-maintenance-lts/action.yml | 1 - .../node/oldest-maintenance-lts/action.yml | 1 - .github/workflows/appsec.yml | 3 ++- .github/workflows/plugins.yml | 1 - .github/workflows/profiling.yml | 2 ++ .github/workflows/tracing.yml | 2 ++ 9 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index f75fe7aeb44..af237f94b14 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -1,8 +1,25 @@ name: Install dependencies description: Install dependencies +inputs: + cache: + description: "Whether to enable caching of node_modules." + required: false + default: 'false' runs: using: composite - steps: # retry in case of server error from registry + steps: + - id: yarn-cache + uses: actions/cache@v4 + with: + key: yarn-cache-${{ github.workflow }}-${{ github.job }}-${{ hashFiles('yarn.lock') }} + path: node_modules.tar + if: inputs.cache == 'true' + - run: 7z x -y node_modules.tar + shell: bash + if: inputs.cache == 'true' && steps.yarn-cache.outputs.cache-hit == 'true' + # Retry in case of server error from registry. - run: yarn install --frozen-lockfile --ignore-engines || yarn install --frozen-lockfile --ignore-engines shell: bash - + - run: 7z -mx0 a node_modules.tar node_modules + shell: bash + if: inputs.cache == 'true' && steps.yarn-cache.outputs.cache-hit != 'true' diff --git a/.github/actions/node/active-lts/action.yml b/.github/actions/node/active-lts/action.yml index 3c8338fb6be..a5ed1f8967a 100644 --- a/.github/actions/node/active-lts/action.yml +++ b/.github/actions/node/active-lts/action.yml @@ -5,5 +5,4 @@ runs: steps: - uses: actions/setup-node@v4 with: - cache: yarn node-version: '22' diff --git a/.github/actions/node/latest/action.yml b/.github/actions/node/latest/action.yml index 234ab9b80cc..94484bf20da 100644 --- a/.github/actions/node/latest/action.yml +++ b/.github/actions/node/latest/action.yml @@ -5,5 +5,4 @@ runs: steps: - uses: actions/setup-node@v4 with: - cache: yarn node-version: 'latest' diff --git a/.github/actions/node/newest-maintenance-lts/action.yml b/.github/actions/node/newest-maintenance-lts/action.yml index 355a16f482c..ddd0adbb4b3 100644 --- a/.github/actions/node/newest-maintenance-lts/action.yml +++ b/.github/actions/node/newest-maintenance-lts/action.yml @@ -5,5 +5,4 @@ runs: steps: - uses: actions/setup-node@v4 with: - cache: yarn node-version: '20' diff --git a/.github/actions/node/oldest-maintenance-lts/action.yml b/.github/actions/node/oldest-maintenance-lts/action.yml index 4f388621cbe..e7980e3444e 100644 --- a/.github/actions/node/oldest-maintenance-lts/action.yml +++ b/.github/actions/node/oldest-maintenance-lts/action.yml @@ -5,5 +5,4 @@ runs: steps: - uses: actions/setup-node@v4 with: - cache: yarn node-version: '18' diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 54bb4da2ad5..1c74acb5ced 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -44,6 +44,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/node/active-lts - uses: ./.github/actions/install + with: + cache: 'true' - run: yarn test:appsec:ci - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 @@ -230,7 +232,6 @@ jobs: - uses: ./.github/actions/testagent/start - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 with: - cache: yarn node-version: ${{ matrix.version }} - uses: ./.github/actions/install - run: yarn test:appsec:plugins:ci diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index d908ec66566..64ea8e6a9da 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -877,7 +877,6 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 with: - cache: yarn node-version: '16' - uses: ./.github/actions/install - run: yarn config set ignore-engines true diff --git a/.github/workflows/profiling.yml b/.github/workflows/profiling.yml index 34b5328737d..1a5c9f321ad 100644 --- a/.github/workflows/profiling.yml +++ b/.github/workflows/profiling.yml @@ -49,6 +49,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/node/active-lts - uses: ./.github/actions/install + with: + cache: 'true' - run: yarn test:profiler:ci - run: yarn test:integration:profiler - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 diff --git a/.github/workflows/tracing.yml b/.github/workflows/tracing.yml index 3cf981e2fd2..5754cf64583 100644 --- a/.github/workflows/tracing.yml +++ b/.github/workflows/tracing.yml @@ -44,5 +44,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/node/active-lts - uses: ./.github/actions/install + with: + cache: 'true' - run: yarn test:trace:core:ci - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 From a41d43d5586d4ae237c1cbbc302a86c392a3e399 Mon Sep 17 00:00:00 2001 From: Yun Kim <35776586+Yun-Kim@users.noreply.github.com> Date: Fri, 21 Mar 2025 11:34:50 -0400 Subject: [PATCH 17/32] feat(llmobs): add integration tag to llmobs spans (#5465) * Add integration tag to llmobs spans * Add integration tag to tests --- packages/dd-trace/src/llmobs/plugins/base.js | 2 +- .../src/llmobs/plugins/bedrockruntime.js | 3 +- .../dd-trace/src/llmobs/span_processor.js | 3 ++ packages/dd-trace/src/llmobs/tagger.js | 5 +- .../plugins/aws-sdk/bedrockruntime.spec.js | 2 +- .../llmobs/plugins/langchain/index.spec.js | 50 +++++++++---------- .../llmobs/plugins/openai/openaiv3.spec.js | 12 ++--- .../llmobs/plugins/openai/openaiv4.spec.js | 18 +++---- 8 files changed, 51 insertions(+), 44 deletions(-) diff --git a/packages/dd-trace/src/llmobs/plugins/base.js b/packages/dd-trace/src/llmobs/plugins/base.js index e5c31f04c22..446b7fb313f 100644 --- a/packages/dd-trace/src/llmobs/plugins/base.js +++ b/packages/dd-trace/src/llmobs/plugins/base.js @@ -43,7 +43,7 @@ class LLMObsPlugin extends TracingPlugin { llmobsStorage.enterWith({ span }) ctx.llmobs.parent = parent - this._tagger.registerLLMObsSpan(span, { parent, ...registerOptions }) + this._tagger.registerLLMObsSpan(span, { parent, integration: this.constructor.id, ...registerOptions }) } } diff --git a/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js b/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js index c89e28e0b4b..77ea878d95f 100644 --- a/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js +++ b/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js @@ -55,7 +55,8 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin { modelName: modelName.toLowerCase(), modelProvider: modelProvider.toLowerCase(), kind: 'llm', - name: 'bedrock-runtime.command' + name: 'bedrock-runtime.command', + integration: 'bedrock' }) const requestParams = extractRequestParams(request.params, modelProvider) diff --git a/packages/dd-trace/src/llmobs/span_processor.js b/packages/dd-trace/src/llmobs/span_processor.js index 2624fa7c6dd..397a486c959 100644 --- a/packages/dd-trace/src/llmobs/span_processor.js +++ b/packages/dd-trace/src/llmobs/span_processor.js @@ -7,6 +7,7 @@ const { METADATA, INPUT_MESSAGES, INPUT_VALUE, + INTEGRATION, OUTPUT_MESSAGES, INPUT_DOCUMENTS, OUTPUT_DOCUMENTS, @@ -186,6 +187,8 @@ class LLMObsSpanProcessor { const errType = span.context()._tags[ERROR_TYPE] || error?.name if (errType) tags.error_type = errType if (sessionId) tags.session_id = sessionId + const integration = LLMObsTagger.tagMap.get(span)?.[INTEGRATION] + if (integration) tags.integration = integration const existingTags = LLMObsTagger.tagMap.get(span)?.[TAGS] || {} if (existingTags) tags = { ...tags, ...existingTags } return Object.entries(tags).map(([key, value]) => `${key}:${value ?? ''}`) diff --git a/packages/dd-trace/src/llmobs/tagger.js b/packages/dd-trace/src/llmobs/tagger.js index ae7f0e0e35f..abb230d06bb 100644 --- a/packages/dd-trace/src/llmobs/tagger.js +++ b/packages/dd-trace/src/llmobs/tagger.js @@ -10,6 +10,7 @@ const { INPUT_VALUE, OUTPUT_DOCUMENTS, INPUT_DOCUMENTS, + INTEGRATION, OUTPUT_VALUE, METADATA, METRICS, @@ -51,7 +52,8 @@ class LLMObsTagger { mlApp, parent, kind, - name + name, + integration } = {}) { if (!this._config.llmobs.enabled) return if (!kind) return // do not register it in the map if it doesn't have an llmobs span kind @@ -66,6 +68,7 @@ class LLMObsTagger { sessionId = sessionId || registry.get(parent)?.[SESSION_ID] if (sessionId) this._setTag(span, SESSION_ID, sessionId) + if (integration) this._setTag(span, INTEGRATION, integration) if (!mlApp) mlApp = registry.get(parent)?.[ML_APP] || this._config.llmobs.mlApp this._setTag(span, ML_APP, mlApp) diff --git a/packages/dd-trace/test/llmobs/plugins/aws-sdk/bedrockruntime.spec.js b/packages/dd-trace/test/llmobs/plugins/aws-sdk/bedrockruntime.spec.js index ad2b55dcb13..c5d851fc962 100644 --- a/packages/dd-trace/test/llmobs/plugins/aws-sdk/bedrockruntime.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/aws-sdk/bedrockruntime.spec.js @@ -109,7 +109,7 @@ describe('Plugin', () => { temperature: modelConfig.temperature, max_tokens: modelConfig.maxTokens }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'bedrock' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) diff --git a/packages/dd-trace/test/llmobs/plugins/langchain/index.spec.js b/packages/dd-trace/test/llmobs/plugins/langchain/index.spec.js index c2c0d294953..7ecd272d92e 100644 --- a/packages/dd-trace/test/llmobs/plugins/langchain/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/langchain/index.spec.js @@ -123,7 +123,7 @@ describe('integrations', () => { outputMessages: [{ content: 'Hello, world!' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 8, output_tokens: 12, total_tokens: 20 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -151,7 +151,7 @@ describe('integrations', () => { outputMessages: [{ content: '' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' }, error: 1, errorType: 'Error', errorMessage: MOCK_STRING, @@ -210,7 +210,7 @@ describe('integrations', () => { metadata: MOCK_ANY, // @langchain/cohere does not provide token usage in the response tokenMetrics: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -255,7 +255,7 @@ describe('integrations', () => { outputMessages: [{ content: 'Hello, world!', role: 'assistant' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 8, output_tokens: 12, total_tokens: 20 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -283,7 +283,7 @@ describe('integrations', () => { outputMessages: [{ content: '' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' }, error: 1, errorType: 'Error', errorMessage: MOCK_STRING, @@ -334,7 +334,7 @@ describe('integrations', () => { outputMessages: [{ content: 'Hello!', role: 'assistant' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 11, output_tokens: 6, total_tokens: 17 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -398,7 +398,7 @@ describe('integrations', () => { metadata: MOCK_ANY, // also tests tokens not sent on llm-type spans should be 0 tokenMetrics: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -455,7 +455,7 @@ describe('integrations', () => { inputDocuments: [{ text: 'Hello!' }], outputValue: '[1 embedding(s) returned with size 2]', metadata: MOCK_ANY, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -482,7 +482,7 @@ describe('integrations', () => { inputDocuments: [{ text: 'Hello!' }], outputValue: '', metadata: MOCK_ANY, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' }, error: 1, errorType: 'Error', errorMessage: MOCK_STRING, @@ -533,7 +533,7 @@ describe('integrations', () => { inputDocuments: [{ text: 'Hello!' }, { text: 'World!' }], outputValue: '[2 embedding(s) returned with size 2]', metadata: MOCK_ANY, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -583,7 +583,7 @@ describe('integrations', () => { inputValue: JSON.stringify({ input: 'Can you tell me about LangSmith?' }), outputValue: 'LangSmith can help with testing in several ways.', metadata: MOCK_ANY, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedLLM = expectedLLMObsLLMSpanEvent({ @@ -601,7 +601,7 @@ describe('integrations', () => { outputMessages: [{ content: 'LangSmith can help with testing in several ways.' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 8, output_tokens: 12, total_tokens: 20 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(workflowSpanEvent).to.deepEqualWithMockValues(expectedWorkflow) @@ -631,7 +631,7 @@ describe('integrations', () => { inputValue: 'Hello!', outputValue: '', metadata: MOCK_ANY, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' }, error: 1, errorType: 'Error', errorMessage: MOCK_STRING, @@ -721,7 +721,7 @@ describe('integrations', () => { name: 'langchain_core.runnables.RunnableSequence', inputValue: JSON.stringify({ person: 'Abraham Lincoln', language: 'Spanish' }), outputValue: 'Springfield, Illinois está en los Estados Unidos.', - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedFirstSubWorkflow = expectedLLMObsNonLLMSpanEvent({ @@ -731,7 +731,7 @@ describe('integrations', () => { name: 'langchain_core.runnables.RunnableSequence', inputValue: JSON.stringify({ person: 'Abraham Lincoln', language: 'Spanish' }), outputValue: 'Springfield, Illinois', - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedFirstLLM = expectedLLMObsLLMSpanEvent({ @@ -747,7 +747,7 @@ describe('integrations', () => { outputMessages: [{ content: 'Springfield, Illinois', role: 'assistant' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 8, output_tokens: 12, total_tokens: 20 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedSecondSubWorkflow = expectedLLMObsNonLLMSpanEvent({ @@ -757,7 +757,7 @@ describe('integrations', () => { name: 'langchain_core.runnables.RunnableSequence', inputValue: JSON.stringify({ language: 'Spanish', city: 'Springfield, Illinois' }), outputValue: 'Springfield, Illinois está en los Estados Unidos.', - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedSecondLLM = expectedLLMObsLLMSpanEvent({ @@ -773,7 +773,7 @@ describe('integrations', () => { outputMessages: [{ content: 'Springfield, Illinois está en los Estados Unidos.', role: 'assistant' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 8, output_tokens: 12, total_tokens: 20 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(topLevelWorkflowSpanEvent).to.deepEqualWithMockValues(expectedTopLevelWorkflow) @@ -859,7 +859,7 @@ describe('integrations', () => { 'Why did the chicken cross the road? To get to the other side!', 'Why was the dog confused? It was barking up the wrong tree!' ]), - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedFirstLLM = expectedLLMObsLLMSpanEvent({ @@ -876,7 +876,7 @@ describe('integrations', () => { }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 37, output_tokens: 10, total_tokens: 47 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedSecondLLM = expectedLLMObsLLMSpanEvent({ @@ -893,7 +893,7 @@ describe('integrations', () => { }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 37, output_tokens: 10, total_tokens: 47 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(workflowSpanEvent).to.deepEqualWithMockValues(expectedWorkflow) @@ -963,7 +963,7 @@ describe('integrations', () => { content: 'Mitochondria', role: 'assistant' }), - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedLLM = expectedLLMObsLLMSpanEvent({ @@ -994,7 +994,7 @@ describe('integrations', () => { outputMessages: [{ content: 'Mitochondria', role: 'assistant' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 8, output_tokens: 12, total_tokens: 20 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(workflowSpanEvent).to.deepEqualWithMockValues(expectedWorkflow) @@ -1064,7 +1064,7 @@ describe('integrations', () => { name: 'langchain_core.runnables.RunnableSequence', inputValue: JSON.stringify({ foo: 'bar' }), outputValue: '3 squared is 9', - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) const expectedTask = expectedLLMObsNonLLMSpanEvent({ @@ -1088,7 +1088,7 @@ describe('integrations', () => { outputMessages: [{ content: '3 squared is 9', role: 'assistant' }], metadata: MOCK_ANY, tokenMetrics: { input_tokens: 8, output_tokens: 12, total_tokens: 20 }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'langchain' } }) expect(workflowSpanEvent).to.deepEqualWithMockValues(expectedWorkflow) diff --git a/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js b/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js index f695a784e25..6073966c16d 100644 --- a/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js @@ -99,7 +99,7 @@ describe('integrations', () => { modelName: 'text-davinci-002', modelProvider: 'openai', metadata: {}, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -156,7 +156,7 @@ describe('integrations', () => { modelName: 'gpt-3.5-turbo-0301', modelProvider: 'openai', metadata: {}, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -207,7 +207,7 @@ describe('integrations', () => { modelName: 'text-embedding-ada-002-v2', modelProvider: 'openai', metadata: { encoding_format: 'float' }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -274,7 +274,7 @@ describe('integrations', () => { ] }], metadata: { function_call: 'auto' }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' }, tokenMetrics: { input_tokens: 37, output_tokens: 10, total_tokens: 47 } }) @@ -311,7 +311,7 @@ describe('integrations', () => { modelName: 'gpt-3.5-turbo', modelProvider: 'openai', metadata: { max_tokens: 50 }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' }, error, errorType: error.type || error.name, errorMessage: error.message, @@ -354,7 +354,7 @@ describe('integrations', () => { modelName: 'gpt-3.5-turbo', modelProvider: 'openai', metadata: { max_tokens: 50 }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' }, error, errorType: error.type || error.name, errorMessage: error.message, diff --git a/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js b/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js index 33177509d7d..d5b073f0d3e 100644 --- a/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js @@ -125,7 +125,7 @@ describe('integrations', () => { modelName: 'text-davinci-002', modelProvider: 'openai', metadata: {}, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -181,7 +181,7 @@ describe('integrations', () => { modelName: 'gpt-3.5-turbo-0301', modelProvider: 'openai', metadata: {}, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -231,7 +231,7 @@ describe('integrations', () => { modelName: 'text-embedding-ada-002-v2', modelProvider: 'openai', metadata: { encoding_format: 'float' }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -306,7 +306,7 @@ describe('integrations', () => { ] }], metadata: { tool_choice: 'auto' }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' }, tokenMetrics: { input_tokens: 37, output_tokens: 10, total_tokens: 47 } }) @@ -355,7 +355,7 @@ describe('integrations', () => { modelName: 'text-davinci-002', modelProvider: 'openai', metadata: { temperature: 0.5, stream: true }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -406,7 +406,7 @@ describe('integrations', () => { modelName: 'gpt-3.5-turbo-0301', modelProvider: 'openai', metadata: { stream: true }, - tags: { ml_app: 'test', language: 'javascript' } + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' } }) expect(spanEvent).to.deepEqualWithMockValues(expected) @@ -463,7 +463,7 @@ describe('integrations', () => { ] }], metadata: { tool_choice: 'auto', stream: true }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' }, tokenMetrics: { input_tokens: 9, output_tokens: 5, total_tokens: 14 } }) @@ -507,7 +507,7 @@ describe('integrations', () => { modelName: 'gpt-3.5-turbo', modelProvider: 'openai', metadata: { max_tokens: 50 }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' }, error, errorType: error.type || error.name, errorMessage: error.message, @@ -549,7 +549,7 @@ describe('integrations', () => { modelName: 'gpt-3.5-turbo', modelProvider: 'openai', metadata: { max_tokens: 50 }, - tags: { ml_app: 'test', language: 'javascript' }, + tags: { ml_app: 'test', language: 'javascript', integration: 'openai' }, error, errorType: error.type || error.name, errorMessage: error.message, From bc3be93c6e5c37591f0bdc41ddfd08709392c5fd Mon Sep 17 00:00:00 2001 From: Yun Kim <35776586+Yun-Kim@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:12:45 -0400 Subject: [PATCH 18/32] chore(llmobs): add span.finished telemetry metric (#5444) * Add span.finished telemetry metric * Add decorator tag * address comments * Fix root parent bool * Add error tag * Fix boolean coalescing --- .../dd-trace/src/llmobs/constants/tags.js | 2 + packages/dd-trace/src/llmobs/noop.js | 6 +-- packages/dd-trace/src/llmobs/sdk.js | 2 + .../dd-trace/src/llmobs/span_processor.js | 3 ++ packages/dd-trace/src/llmobs/tagger.js | 9 ++-- packages/dd-trace/src/llmobs/telemetry.js | 49 ++++++++++++++++++- 6 files changed, 64 insertions(+), 7 deletions(-) diff --git a/packages/dd-trace/src/llmobs/constants/tags.js b/packages/dd-trace/src/llmobs/constants/tags.js index eee9a6b9890..889594f87ab 100644 --- a/packages/dd-trace/src/llmobs/constants/tags.js +++ b/packages/dd-trace/src/llmobs/constants/tags.js @@ -4,6 +4,8 @@ module.exports = { SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval'], SPAN_KIND: '_ml_obs.meta.span.kind', SESSION_ID: '_ml_obs.session_id', + DECORATOR: '_ml_obs.decorator', + INTEGRATION: '_ml_obs.integration', METADATA: '_ml_obs.meta.metadata', METRICS: '_ml_obs.metrics', ML_APP: '_ml_obs.meta.ml_app', diff --git a/packages/dd-trace/src/llmobs/noop.js b/packages/dd-trace/src/llmobs/noop.js index 4eba48cd51c..d25790cd34c 100644 --- a/packages/dd-trace/src/llmobs/noop.js +++ b/packages/dd-trace/src/llmobs/noop.js @@ -43,14 +43,14 @@ class NoopLLMObs { const ctx = ctxOrPropertyKey if (ctx.kind !== 'method') return target - return llmobs.wrap({ name: ctx.name, ...options }, target) + return llmobs.wrap({ name: ctx.name, _decorator: true, ...options }, target) } else { const propertyKey = ctxOrPropertyKey if (descriptor) { if (typeof descriptor.value !== 'function') return descriptor const original = descriptor.value - descriptor.value = llmobs.wrap({ name: propertyKey, ...options }, original) + descriptor.value = llmobs.wrap({ name: propertyKey, _decorator: true, ...options }, original) return descriptor } else { @@ -59,7 +59,7 @@ class NoopLLMObs { const original = target[propertyKey] Object.defineProperty(target, propertyKey, { ...Object.getOwnPropertyDescriptor(target, propertyKey), - value: llmobs.wrap({ name: propertyKey, ...options }, original) + value: llmobs.wrap({ name: propertyKey, _decorator: true, ...options }, original) }) return target diff --git a/packages/dd-trace/src/llmobs/sdk.js b/packages/dd-trace/src/llmobs/sdk.js index f14ee698d09..41a09c5006d 100644 --- a/packages/dd-trace/src/llmobs/sdk.js +++ b/packages/dd-trace/src/llmobs/sdk.js @@ -430,6 +430,7 @@ class LLMObs extends NoopLLMObs { modelProvider, sessionId, mlApp, + _decorator, ...spanOptions } = options @@ -438,6 +439,7 @@ class LLMObs extends NoopLLMObs { modelName, modelProvider, sessionId, + _decorator, spanOptions } } diff --git a/packages/dd-trace/src/llmobs/span_processor.js b/packages/dd-trace/src/llmobs/span_processor.js index 397a486c959..c719463e5dd 100644 --- a/packages/dd-trace/src/llmobs/span_processor.js +++ b/packages/dd-trace/src/llmobs/span_processor.js @@ -27,6 +27,8 @@ const { ERROR_STACK } = require('../constants') +const telemetry = require('./telemetry') + const LLMObsTagger = require('./tagger') const tracerVersion = require('../../../../package.json').version @@ -49,6 +51,7 @@ class LLMObsSpanProcessor { try { const formattedEvent = this.format(span) + telemetry.incrementLLMObsSpanFinishedCount(span) this._writer.append(formattedEvent) } catch (e) { // this should be a rare case diff --git a/packages/dd-trace/src/llmobs/tagger.js b/packages/dd-trace/src/llmobs/tagger.js index abb230d06bb..b20355d5a75 100644 --- a/packages/dd-trace/src/llmobs/tagger.js +++ b/packages/dd-trace/src/llmobs/tagger.js @@ -10,7 +10,6 @@ const { INPUT_VALUE, OUTPUT_DOCUMENTS, INPUT_DOCUMENTS, - INTEGRATION, OUTPUT_VALUE, METADATA, METRICS, @@ -23,7 +22,9 @@ const { ROOT_PARENT_ID, INPUT_TOKENS_METRIC_KEY, OUTPUT_TOKENS_METRIC_KEY, - TOTAL_TOKENS_METRIC_KEY + TOTAL_TOKENS_METRIC_KEY, + INTEGRATION, + DECORATOR } = require('./constants/tags') // global registry of LLMObs spans @@ -53,7 +54,8 @@ class LLMObsTagger { parent, kind, name, - integration + integration, + _decorator } = {}) { if (!this._config.llmobs.enabled) return if (!kind) return // do not register it in the map if it doesn't have an llmobs span kind @@ -69,6 +71,7 @@ class LLMObsTagger { sessionId = sessionId || registry.get(parent)?.[SESSION_ID] if (sessionId) this._setTag(span, SESSION_ID, sessionId) if (integration) this._setTag(span, INTEGRATION, integration) + if (_decorator) this._setTag(span, DECORATOR, _decorator) if (!mlApp) mlApp = registry.get(parent)?.[ML_APP] || this._config.llmobs.mlApp this._setTag(span, ML_APP, mlApp) diff --git a/packages/dd-trace/src/llmobs/telemetry.js b/packages/dd-trace/src/llmobs/telemetry.js index aa6b06ec074..b726bb6295b 100644 --- a/packages/dd-trace/src/llmobs/telemetry.js +++ b/packages/dd-trace/src/llmobs/telemetry.js @@ -1,12 +1,59 @@ 'use strict' +const { + SPAN_KIND, + MODEL_PROVIDER, + PARENT_ID_KEY, + SESSION_ID, + ROOT_PARENT_ID, + INTEGRATION, + DECORATOR +} = require('./constants/tags') + +const ERROR_TYPE = require('../constants') + const telemetryMetrics = require('../telemetry/metrics') + +const LLMObsTagger = require('./tagger') + const llmobsMetrics = telemetryMetrics.manager.namespace('mlobs') function incrementLLMObsSpanStartCount (tags, value = 1) { llmobsMetrics.count('span.start', tags).inc(value) } +function incrementLLMObsSpanFinishedCount (span, value = 1) { + const mlObsTags = LLMObsTagger.tagMap.get(span) + const spanTags = span.context()._tags + + const isRootSpan = mlObsTags[PARENT_ID_KEY] === ROOT_PARENT_ID + const hasSessionId = mlObsTags[SESSION_ID] != null + const integration = mlObsTags[INTEGRATION] + const autoInstrumented = integration != null + const decorator = !!mlObsTags[DECORATOR] + const spanKind = mlObsTags[SPAN_KIND] + const modelProvider = mlObsTags[MODEL_PROVIDER] + const error = spanTags.error || spanTags[ERROR_TYPE] + + const tags = { + autoinstrumented: Number(autoInstrumented), + has_session_id: Number(hasSessionId), + is_root_span: Number(isRootSpan), + span_kind: spanKind, + integration: integration || 'N/A', + error: error ? 1 : 0 + } + if (!autoInstrumented) { + tags.decorator = Number(decorator) + } + if (modelProvider) { + tags.model_provider = modelProvider + } + + llmobsMetrics.count('span.finished', tags).inc(value) +} + module.exports = { - incrementLLMObsSpanStartCount + incrementLLMObsSpanStartCount, + incrementLLMObsSpanFinishedCount } From acbb8dc971880ee3aab4af7e1d98efe3a5b6e956 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Fri, 21 Mar 2025 18:45:16 -0400 Subject: [PATCH 19/32] fix release script esm error on node 20 (#5368) * use old version of dependency instead of vendoring --- package.json | 2 +- scripts/release/helpers/requirements.js | 2 +- yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index c66216adaac..8d645c06e74 100644 --- a/package.json +++ b/package.json @@ -125,7 +125,7 @@ "@msgpack/msgpack": "^3.0.0-beta3", "@stylistic/eslint-plugin-js": "^3.0.1", "@types/node": "^16.0.0", - "application-config-path": "^1.0.0", + "application-config-path": "^0.1.1", "autocannon": "^4.5.2", "aws-sdk": "^2.1446.0", "axios": "^1.8.2", diff --git a/scripts/release/helpers/requirements.js b/scripts/release/helpers/requirements.js index cd49a200689..e313749be01 100644 --- a/scripts/release/helpers/requirements.js +++ b/scripts/release/helpers/requirements.js @@ -4,7 +4,7 @@ const { join } = require('path') const { existsSync, readFileSync } = require('fs') -const { default: getApplicationConfigPath } = require('application-config-path') +const getApplicationConfigPath = require('application-config-path') const { capture, fatal, run } = require('./terminal') // Check that the `git` CLI is installed. diff --git a/yarn.lock b/yarn.lock index c76f9fa8d99..d8374278a70 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1084,10 +1084,10 @@ append-transform@^2.0.0: dependencies: default-require-extensions "^3.0.0" -application-config-path@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/application-config-path/-/application-config-path-1.0.0.tgz#9c25b8c00ac9a342db27275abd3f38c67bbe5a05" - integrity sha512-6ZDlLTlfqrTybVzZJDpX2K2ZufqyMyiTbOG06GpxmkmczFgTN+YYRGcTcMCXv/F5P5SrZijVjzzpPUE9BvheLg== +application-config-path@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/application-config-path/-/application-config-path-0.1.1.tgz#8b5ac64ff6afdd9bd70ce69f6f64b6998f5f756e" + integrity sha512-zy9cHePtMP0YhwG+CfHm0bgwdnga2X3gZexpdCwEj//dpb+TKajtiC8REEUJUSq6Ab4f9cgNy2l8ObXzCXFkEw== archy@^1.0.0: version "1.0.0" From 7eda6c53e70c9dd9b89477682c3e8b103cfaf353 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Fri, 21 Mar 2025 19:10:34 -0400 Subject: [PATCH 20/32] fix connection pool error in mongoose tests (#5435) --- packages/datadog-plugin-mongoose/test/index.spec.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/datadog-plugin-mongoose/test/index.spec.js b/packages/datadog-plugin-mongoose/test/index.spec.js index 07ef6abc8f4..2ec813f9ce9 100644 --- a/packages/datadog-plugin-mongoose/test/index.spec.js +++ b/packages/datadog-plugin-mongoose/test/index.spec.js @@ -29,11 +29,11 @@ describe('Plugin', () => { }) } - before(() => { + beforeEach(() => { return agent.load(['mongodb-core']) }) - before(async () => { + beforeEach(async () => { tracer = require('../../dd-trace') mongoose = require(`../../../versions/mongoose@${version}`).get() @@ -43,11 +43,11 @@ describe('Plugin', () => { await connect() }) - after(async () => { + afterEach(async () => { return await mongoose.disconnect() }) - after(() => { + afterEach(() => { return agent.close({ ritmReset: false }) }) From ac08612e839aa022bfa8e45bc6eb37918e5c064b Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Mon, 24 Mar 2025 17:37:47 -0400 Subject: [PATCH 21/32] make cache action unix compatible and update test agent (#5464) * use tar for everything * update test-agent to 1.21.1 --- .github/actions/install/action.yml | 6 +++--- .github/workflows/plugins.yml | 2 +- docker-compose.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index af237f94b14..962aecfb968 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -11,15 +11,15 @@ runs: - id: yarn-cache uses: actions/cache@v4 with: - key: yarn-cache-${{ github.workflow }}-${{ github.job }}-${{ hashFiles('yarn.lock') }} + key: yarn-cache-${{ github.workflow }}-${{ github.job }}-${{ hashFiles('yarn.lock') }}-v2 path: node_modules.tar if: inputs.cache == 'true' - - run: 7z x -y node_modules.tar + - run: tar -xf node_modules.tar shell: bash if: inputs.cache == 'true' && steps.yarn-cache.outputs.cache-hit == 'true' # Retry in case of server error from registry. - run: yarn install --frozen-lockfile --ignore-engines || yarn install --frozen-lockfile --ignore-engines shell: bash - - run: 7z -mx0 a node_modules.tar node_modules + - run: tar -cf node_modules.tar node_modules shell: bash if: inputs.cache == 'true' && steps.yarn-cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 64ea8e6a9da..b8eebd4fdbc 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -853,7 +853,7 @@ jobs: - 1521:1521 - 5500:5500 testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0 + image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.21.1 env: LOG_LEVEL: DEBUG TRACE_LANGUAGE: javascript diff --git a/docker-compose.yml b/docker-compose.yml index cebd93ba020..8744e76eb0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -155,7 +155,7 @@ services: - LDAP_PASSWORDS=password1,password2 testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0 + image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.21.1 ports: - "127.0.0.1:9126:9126" environment: From 4bd1de053127546beef121dc653752ead4afd52f Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Mon, 24 Mar 2025 22:25:33 -0400 Subject: [PATCH 22/32] fix wrong test error message when expected span was not received (#5447) --- packages/dd-trace/test/plugins/helpers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/dd-trace/test/plugins/helpers.js b/packages/dd-trace/test/plugins/helpers.js index a320d02681a..65fe42bf764 100644 --- a/packages/dd-trace/test/plugins/helpers.js +++ b/packages/dd-trace/test/plugins/helpers.js @@ -1,6 +1,7 @@ 'use strict' const { AssertionError } = require('assert') +const { inspect } = require('util') const { AsyncResource } = require('../../../datadog-instrumentations/src/helpers/instrument') const Nomenclature = require('../../src/service-naming') @@ -33,7 +34,7 @@ function expectSomeSpan (agent, expected, timeout) { const error = scoredErrors.sort((a, b) => a.score - b.score)[0].err // We'll append all the spans to this error message so it's visible in test // output. - error.message += '\n\nCandidate Traces:\n' + JSON.stringify(traces, null, 2) + error.message += '\n\nCandidate Traces:\n' + inspect(traces) throw error }, timeout) } From 7c5cecd992c3810fbe9a8571c390d3d354a8cb16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= <60353145+Mariovido@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:48:20 +0100 Subject: [PATCH 23/32] [test optimization] [SDTEST-1630] Attempt to fix flaky tests implementation (#5429) --- .../attempt-to-fix.feature | 4 + .../features-test-management/support/steps.js | 15 + .../attempt-to-fix-test.js | 27 ++ .../test-management/test-attempt-to-fix-1.js | 21 ++ .../test-management/test-attempt-to-fix-2.js | 9 + .../vitest-tests/test-attempt-to-fix.mjs | 22 ++ integration-tests/cucumber/cucumber.spec.js | 228 +++++++++++++++- integration-tests/cypress/cypress.spec.js | 245 ++++++++++++++++- .../cypress/e2e/attempt-to-fix.js | 20 ++ integration-tests/jest/jest.spec.js | 256 +++++++++++++++++- integration-tests/mocha/mocha.spec.js | 226 +++++++++++++++- .../playwright/playwright.spec.js | 228 +++++++++++++++- integration-tests/vitest/vitest.spec.js | 220 ++++++++++++++- .../datadog-instrumentations/src/cucumber.js | 84 ++++-- packages/datadog-instrumentations/src/jest.js | 182 +++++++++---- .../src/mocha/main.js | 24 +- .../src/mocha/utils.js | 119 ++++++-- .../src/mocha/worker.js | 4 +- .../src/playwright.js | 114 ++++++-- .../datadog-instrumentations/src/vitest.js | 88 +++++- packages/datadog-plugin-cucumber/src/index.js | 25 +- .../src/cypress-plugin.js | 82 +++++- .../datadog-plugin-cypress/src/support.js | 49 +++- packages/datadog-plugin-jest/src/index.js | 43 ++- packages/datadog-plugin-mocha/src/index.js | 33 ++- .../datadog-plugin-playwright/src/index.js | 24 +- packages/datadog-plugin-vitest/src/index.js | 51 +++- .../exporters/ci-visibility-exporter.js | 7 +- .../requests/get-library-configuration.js | 4 +- .../get-test-management-tests.js | 6 +- packages/dd-trace/src/plugins/ci_plugin.js | 15 +- packages/dd-trace/src/plugins/util/test.js | 23 +- 32 files changed, 2300 insertions(+), 198 deletions(-) create mode 100644 integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature create mode 100644 integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js create mode 100644 integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js create mode 100644 integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js create mode 100644 integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs create mode 100644 integration-tests/cypress/e2e/attempt-to-fix.js diff --git a/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature b/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature new file mode 100644 index 00000000000..98ac42a8213 --- /dev/null +++ b/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature @@ -0,0 +1,4 @@ +Feature: Attempt to fix + Scenario: Say attempt to fix + When the greeter says attempt to fix + Then I should have heard "attempt to fix" diff --git a/integration-tests/ci-visibility/features-test-management/support/steps.js b/integration-tests/ci-visibility/features-test-management/support/steps.js index e01c21e968a..67a2ed51361 100644 --- a/integration-tests/ci-visibility/features-test-management/support/steps.js +++ b/integration-tests/ci-visibility/features-test-management/support/steps.js @@ -1,6 +1,8 @@ const assert = require('assert') const { When, Then } = require('@cucumber/cucumber') +let numAttempt = 0 + Then('I should have heard {string}', function (expectedResponse) { if (this.whatIHeard === 'quarantine') { assert.equal(this.whatIHeard, 'fail') @@ -21,3 +23,16 @@ When('the greeter says disabled', function () { // expected to fail if not disabled this.whatIHeard = 'disabld' }) + +When('the greeter says attempt to fix', function () { + // eslint-disable-next-line no-console + console.log('I am running') // just to assert whether this is running + // expected to fail + if (process.env.SHOULD_ALWAYS_PASS) { + this.whatIHeard = 'attempt to fix' + } else if (process.env.SHOULD_FAIL_SOMETIMES) { + this.whatIHeard = numAttempt++ % 2 === 0 ? 'attempt to fix' : 'attempt to fx' + } else { + this.whatIHeard = 'attempt to fx' + } +}) diff --git a/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js b/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js new file mode 100644 index 00000000000..f235d10f549 --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js @@ -0,0 +1,27 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test.describe('attempt to fix', () => { + test('should attempt to fix failed test', async ({ page }) => { + let textToAssert + + if (process.env.SHOULD_ALWAYS_PASS) { + textToAssert = 'Hello World' + } else if (process.env.SHOULD_FAIL_SOMETIMES) { + // can't use numAttempt++ because we're running in parallel + if (Number(process.env.TEST_WORKER_INDEX) % 2 === 0) { + throw new Error('Hello Warld') + } + textToAssert = 'Hello World' + } else { + textToAssert = 'Hello Warld' + } + + await expect(page.locator('.hello-world')).toHaveText([ + textToAssert + ]) + }) +}) diff --git a/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js new file mode 100644 index 00000000000..be05f47fd50 --- /dev/null +++ b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js @@ -0,0 +1,21 @@ +const { expect } = require('chai') + +let numAttempts = 0 + +describe('attempt to fix tests', () => { + it('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running when attempt to fix') // to check if this is being run + if (process.env.SHOULD_ALWAYS_PASS) { + expect(1 + 2).to.equal(3) + } else if (process.env.SHOULD_FAIL_SOMETIMES) { + if (numAttempts++ % 2 === 0) { + expect(1 + 2).to.equal(3) + } else { + expect(1 + 2).to.equal(4) + } + } else { + expect(1 + 2).to.equal(4) + } + }) +}) diff --git a/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js new file mode 100644 index 00000000000..053d1d62eb0 --- /dev/null +++ b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js @@ -0,0 +1,9 @@ +const { expect } = require('chai') + +describe('attempt to fix tests 2', () => { + it('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running when attempt to fix 2') // to check if this is being run + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs b/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs new file mode 100644 index 00000000000..4fe4ba6cacc --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs @@ -0,0 +1,22 @@ +import { describe, test, expect } from 'vitest' + +let numAttempt = 0 + +describe('attempt to fix tests', () => { + test('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running') // to check if this is being run + if (process.env.SHOULD_ALWAYS_PASS) { + expect(1 + 2).to.equal(3) + } else if (process.env.SHOULD_FAIL_SOMETIMES) { + // We need the last attempt to fail for the exit code to be 1 + if (numAttempt++ % 2 === 1) { + expect(1 + 2).to.equal(4) + } else { + expect(1 + 2).to.equal(3) + } + } else { + expect(1 + 2).to.equal(4) + } + }) +}) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 2359577b9ea..092dcbe3af7 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -49,7 +49,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -2031,6 +2034,229 @@ versions.forEach(version => { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + cucumber: { + suites: { + 'ci-visibility/features-test-management/attempt-to-fix.feature': { + tests: { + 'Say attempt to fix': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptToFix, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'Say attempt to fix' + ) + + if (isAttemptToFix) { + // 3 retries + 1 initial run + assert.equal(retriedTests.length, 4) + } else { + assert.equal(retriedTests.length, 1) + } + + for (let i = 0; i < retriedTests.length; i++) { + const isFirstAttempt = i === 0 + const isLastAttempt = i === retriedTests.length - 1 + const test = retriedTests[i] + + assert.equal( + test.resource, + 'ci-visibility/features-test-management/attempt-to-fix.feature.Say attempt to fix' + ) + + if (isDisabled) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + } else if (isQuarantined) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_DISABLED) + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_QUARANTINED) + } + + if (isAttemptToFix) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (!isFirstAttempt) { + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } + if (isLastAttempt) { + if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + } else if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + } + }) + + const runTest = (done, { + isAttemptToFix, + isQuarantined, + isDisabled, + extraEnvVars, + shouldAlwaysPass, + shouldFailSometimes + } = {}) => { + const testAssertions = getTestAssertions({ + isAttemptToFix, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes + }) + let stdout = '' + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-test-management/attempt-to-fix.feature', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + ...extraEnvVars, + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data.toString() + }) + + childProcess.on('exit', exitCode => { + testAssertions.then(() => { + assert.include(stdout, 'I am running') + if (isQuarantined || isDisabled || shouldAlwaysPass) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, { isAttemptToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, { isAttemptToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, { isAttemptToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, { + extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } + }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cucumber: { + suites: { + 'ci-visibility/features-test-management/attempt-to-fix.feature': { + tests: { + 'Say attempt to fix': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runTest(done, { + isAttemptToFix: true, + isQuarantined: true + }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cucumber: { + suites: { + 'ci-visibility/features-test-management/attempt-to-fix.feature': { + tests: { + 'Say attempt to fix': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runTest(done, { + isAttemptToFix: true, + isDisabled: true + }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 76958fbdac0..1ab83ca8064 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -44,7 +44,11 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_NAME, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -1775,6 +1779,245 @@ moduleTypes.forEach(({ }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + cypress: { + suites: { + 'cypress/e2e/attempt-to-fix.js': { + tests: { + 'attempt to fix is attempt to fix': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'cypress/e2e/attempt-to-fix.js.attempt to fix is attempt to fix' + ] + ) + + const attemptToFixTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix is attempt to fix' + ) + + if (isAttemptToFix) { + assert.equal(attemptToFixTests.length, 4) + } else { + assert.equal(attemptToFixTests.length, 1) + } + + for (let i = attemptToFixTests.length - 1; i >= 0; i--) { + const test = attemptToFixTests[i] + if (!isAttemptToFix) { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + continue + } + if (isQuarantined) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + assert.notPropertyVal(test.meta, TEST_STATUS, 'skip') + } + if (isDisabled) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + assert.notPropertyVal(test.meta, TEST_STATUS, 'skip') + } + + const isLastAttempt = i === attemptToFixTests.length - 1 + const isFirstAttempt = i === 0 + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (isFirstAttempt) { + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } else { + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } + if (isLastAttempt) { + if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } else if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } + } + } + }) + + const runAttemptToFixTest = (done, { + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled, + extraEnvVars = {} + } = {}) => { + const testAssertionsPromise = getTestAssertions({ + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled + }) + + const { + NODE_OPTIONS, + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const specToRun = 'cypress/e2e/attempt-to-fix.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + ...extraEnvVars, + ...(shouldAlwaysPass ? { CYPRESS_SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { CYPRESS_SHOULD_FAIL_SOMETIMES: '1' } : {}) + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (shouldAlwaysPass) { + assert.equal(exitCode, 0) + } else { + // TODO: we need to figure out how to trick cypress into returning exit code 0 + // even if there are failed tests + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + /** + * TODO: + * The spec says that quarantined tests that are not attempted to fix should be run and their result ignored. + * Cypress will skip the test instead. + * + * When a test is quarantined and attempted to fix, the spec is to run the test and ignore its result. + * Cypress will run the test, but it won't ignore its result. + */ + it('can mark tests as quarantined and tests are not skipped', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cypress: { + suites: { + 'cypress/e2e/attempt-to-fix.js': { + tests: { + 'attempt to fix is attempt to fix': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isQuarantined: true }) + }) + + /** + * TODO: + * When a test is disabled and attempted to fix, the spec is to run the test and ignore its result. + * Cypress will run the test, but it won't ignore its result. + */ + it('can mark tests as disabled and tests are not skipped', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cypress: { + suites: { + 'cypress/e2e/attempt-to-fix.js': { + tests: { + 'attempt to fix is attempt to fix': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isDisabled: true }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/cypress/e2e/attempt-to-fix.js b/integration-tests/cypress/e2e/attempt-to-fix.js new file mode 100644 index 00000000000..9c638c4b6dd --- /dev/null +++ b/integration-tests/cypress/e2e/attempt-to-fix.js @@ -0,0 +1,20 @@ +/* eslint-disable */ + +let numAttempt = 0 + +function getTextToAssert () { + if (Cypress.env('SHOULD_ALWAYS_PASS')) { + return 'Hello World' + } else if (Cypress.env('SHOULD_FAIL_SOMETIMES')) { + return numAttempt++ % 2 === 0 ? 'Hello World' : 'Hello Warld' + } + return 'Hello Warld' +} + +describe('attempt to fix', () => { + it('is attempt to fix', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', getTextToAssert()) + }) +}) diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index 06708e793eb..f2352437d2a 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -46,7 +46,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2942,6 +2945,257 @@ describe('jest CommonJS', () => { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + jest: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptToFix, + isParallel, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-management/test-attempt-to-fix-1.js.attempt to fix tests can attempt to fix a test' + ] + ) + + if (isParallel) { + // Parallel mode in jest requires more than a single test suite + // Here we check that the second test suite is actually running, + // so we can be sure that parallel mode is on + const parallelTestName = 'ci-visibility/test-management/test-attempt-to-fix-2.js.' + + 'attempt to fix tests 2 can attempt to fix a test' + assert.includeMembers(resourceNames, [parallelTestName]) + } + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + if (!isAttemptToFix) { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + continue + } + + if (isQuarantined) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } + + if (isDisabled) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + } + + const isFirstAttempt = i === 0 + const isLastAttempt = i === retriedTests.length - 1 + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + + if (isFirstAttempt) { + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } else { + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } + + if (isLastAttempt) { + if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } else if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + } + }) + + const runAttemptToFixTest = (done, { + isAttemptToFix, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes, + extraEnvVars = {}, + isParallel = false + } = {}) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions({ + isAttemptToFix, + isParallel, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test-management/test-attempt-to-fix-1', + SHOULD_CHECK_RESULTS: '1', + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}), + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.stderr.on('data', (chunk) => { + stdout += chunk.toString() + }) + + childProcess.stdout.on('data', (chunk) => { + stdout += chunk.toString() + }) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running when attempt to fix') + if (isQuarantined || shouldAlwaysPass || isDisabled) { + // even though a test fails, the exit code is 0 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + jest: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isQuarantined: true }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + jest: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isDisabled: true }) + }) + + it('can attempt to fix in parallel mode', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest( + done, + { + isAttemptToFix: true, + isParallel: true, + extraEnvVars: { + // we need to run more than 1 suite for parallel mode to kick in + TESTS_TO_RUN: 'test-management/test-attempt-to-fix', + RUN_IN_PARALLEL: true + } + } + ) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index 6c96d2fb136..c96d99953c4 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -48,7 +48,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2566,6 +2569,227 @@ describe('mocha CommonJS', function () { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-management/test-attempt-to-fix-1.js.attempt to fix tests can attempt to fix a test' + ] + ) + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + const isFirstAttempt = i === 0 + const isLastAttempt = i === retriedTests.length - 1 + if (!isAttemptToFix) { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + continue + } + + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (isFirstAttempt) { + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } else { + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } + + if (isQuarantined) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } + + if (isDisabled) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + } + + if (isLastAttempt) { + if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + } else if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } + } + } + }) + + const runAttemptToFixTest = (done, { + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled, + extraEnvVars = {} + } = {}) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions({ + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-management/test-attempt-to-fix-1.js' + ]), + SHOULD_CHECK_RESULTS: '1', + ...extraEnvVars, + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data + }) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running when attempt to fix') + if (shouldAlwaysPass || isQuarantined || isDisabled) { + // even though a test fails, the exit code is 0 because the test is quarantined or disabled + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isQuarantined: true }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isDisabled: true }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 0ed40cdfe7d..347ae5f980d 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -33,7 +33,11 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_NAME, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -898,6 +902,228 @@ versions.forEach((version) => { if (version === 'latest') { context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + playwright: { + suites: { + 'attempt-to-fix-test.js': { + tests: { + 'attempt to fix should attempt to fix failed test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptingToFix, + shouldAlwaysPass, + shouldFailSometimes, + isDisabled, + isQuarantined + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptingToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const attemptedToFixTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix should attempt to fix failed test' + ) + + if (isAttemptingToFix) { + assert.equal(attemptedToFixTests.length, 4) + } else { + assert.equal(attemptedToFixTests.length, 1) + } + + if (isDisabled) { + const numDisabledTests = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_IS_DISABLED] === 'true' + ).length + assert.equal(numDisabledTests, attemptedToFixTests.length) + } + + if (isQuarantined) { + const numQuarantinedTests = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_IS_QUARANTINED] === 'true' + ).length + assert.equal(numQuarantinedTests, attemptedToFixTests.length) + } + + // Retried tests are in randomly order, so we just count number of tests + const countAttemptToFixTests = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true' + ).length + + const countRetriedAttemptToFixTests = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true' && + test.meta[TEST_IS_RETRY] === 'true' && + test.meta[TEST_RETRY_REASON] === 'attempt_to_fix' + ).length + + const testsMarkedAsFailedAllRetries = attemptedToFixTests.filter(test => + test.meta[TEST_HAS_FAILED_ALL_RETRIES] === 'true' + ).length + + const testsMarkedAsPassedAllRetries = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED] === 'true' + ).length + + if (isAttemptingToFix) { + assert.equal(countAttemptToFixTests, attemptedToFixTests.length) + assert.equal(countRetriedAttemptToFixTests, attemptedToFixTests.length - 1) + if (shouldAlwaysPass) { + assert.equal(testsMarkedAsFailedAllRetries, 0) + assert.equal(testsMarkedAsPassedAllRetries, 1) + } else if (shouldFailSometimes) { + assert.equal(testsMarkedAsFailedAllRetries, 0) + assert.equal(testsMarkedAsPassedAllRetries, 0) + } else { // always fail + assert.equal(testsMarkedAsFailedAllRetries, 1) + assert.equal(testsMarkedAsPassedAllRetries, 0) + } + } else { + assert.equal(countAttemptToFixTests, 0) + assert.equal(countRetriedAttemptToFixTests, 0) + assert.equal(testsMarkedAsFailedAllRetries, 0) + assert.equal(testsMarkedAsPassedAllRetries, 0) + } + }) + + const runAttemptToFixTest = (done, { + isAttemptingToFix, + isQuarantined, + extraEnvVars, + shouldAlwaysPass, + shouldFailSometimes, + isDisabled + } = {}) => { + const testAssertionsPromise = getTestAssertions({ + isAttemptingToFix, + shouldAlwaysPass, + shouldFailSometimes, + isDisabled, + isQuarantined + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js attempt-to-fix-test.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-tests-test-management', + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}), + ...extraEnvVars + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (isQuarantined || isDisabled || shouldAlwaysPass) { + // even though a test fails, the exit code is 0 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + playwright: { + suites: { + 'attempt-to-fix-test.js': { + tests: { + 'attempt to fix should attempt to fix failed test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, isQuarantined: true }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + playwright: { + suites: { + 'attempt-to-fix-test.js': { + tests: { + 'attempt to fix should attempt to fix failed test': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, isDisabled: true }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index a4d68ce87b8..3e26a1c9d03 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -37,7 +37,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -1341,6 +1344,217 @@ versions.forEach((version) => { if (version === 'latest') { context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptingToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantining, + isDisabling + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptingToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs.attempt to fix tests can attempt to fix a test' + ] + ) + + const attemptedToFixTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < attemptedToFixTests.length; i++) { + const isFirstAttempt = i === 0 + const isLastAttempt = i === attemptedToFixTests.length - 1 + const test = attemptedToFixTests[i] + if (isQuarantining) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else if (isDisabling) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + } + + if (isAttemptingToFix) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (isFirstAttempt) { + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + continue + } + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + if (isLastAttempt) { + if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } else if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + } + }) + + const runAttemptToFixTest = (done, { + isAttemptingToFix, + shouldAlwaysPass, + isQuarantining, + shouldFailSometimes, + isDisabling, + extraEnvVars = {} + } = {}) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions({ + isAttemptingToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantining, + isDisabling + }) + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/test-attempt-to-fix*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init --no-warnings', + ...extraEnvVars, + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data + }) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running') + if (shouldAlwaysPass || (isAttemptingToFix && isQuarantining) || (isAttemptingToFix && isDisabling)) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, isQuarantining: true }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, isDisabling: true }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ @@ -1428,7 +1642,7 @@ versions.forEach((version) => { assert.equal(exitCode, 1) } done() - }) + }).catch(done) }) } @@ -1541,7 +1755,7 @@ versions.forEach((version) => { assert.equal(exitCode, 1) } done() - }) + }).catch(done) }) } diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index ce82c268e3f..bc7bc55b932 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -73,6 +73,7 @@ let isEarlyFlakeDetectionFaulty = false let isFlakyTestRetriesEnabled = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false +let testManagementAttemptToFixRetries = 0 let testManagementTests = {} let numTestRetries = 0 let knownTests = [] @@ -121,10 +122,10 @@ function isNewTest (testSuite, testName) { } function getTestProperties (testSuite, testName) { - const { disabled, quarantined } = + const { attempt_to_fix: attemptToFix, disabled, quarantined } = testManagementTests?.cucumber?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { disabled, quarantined } + return { attemptToFix, disabled, quarantined } } function getTestStatusFromRetries (testStatuses) { @@ -303,22 +304,42 @@ function wrapRun (pl, isLatestVersion) { } let isNew = false let isEfdRetry = false + let isAttemptToFix = false + let isAttemptToFixRetry = false + let hasFailedAllRetries = false + let hasPassedAllRetries = false let isDisabled = false let isQuarantined = false - if (isKnownTestsEnabled && status !== 'skip') { - const numRetries = numRetriesByPickleId.get(this.pickle.id) - isNew = numRetries !== undefined - isEfdRetry = numRetries > 0 - } if (isTestManagementTestsEnabled) { const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd()) const testProperties = getTestProperties(testSuitePath, this.pickle.name) + const numRetries = numRetriesByPickleId.get(this.pickle.id) + isAttemptToFix = testProperties.attemptToFix + isAttemptToFixRetry = isAttemptToFix && numRetries > 0 isDisabled = testProperties.disabled - if (!isDisabled) { - isQuarantined = testProperties.quarantined + isQuarantined = testProperties.quarantined + + if (isAttemptToFixRetry) { + const statuses = lastStatusByPickleId.get(this.pickle.id) + if (statuses.length === testManagementAttemptToFixRetries + 1) { + const { pass, fail } = statuses.reduce((acc, status) => { + acc[status]++ + return acc + }, { pass: 0, fail: 0 }) + hasFailedAllRetries = fail === testManagementAttemptToFixRetries + 1 + hasPassedAllRetries = pass === testManagementAttemptToFixRetries + 1 + } } } + + if (isKnownTestsEnabled && status !== 'skip' && !isAttemptToFix) { + const numRetries = numRetriesByPickleId.get(this.pickle.id) + + isNew = numRetries !== undefined + isEfdRetry = numRetries > 0 + } + const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) const error = getErrorFromCucumberResult(result) @@ -334,6 +355,10 @@ function wrapRun (pl, isLatestVersion) { isNew, isEfdRetry, isFlakyRetry: numAttempt > 0, + isAttemptToFix, + isAttemptToFixRetry, + hasFailedAllRetries, + hasPassedAllRetries, isDisabled, isQuarantined }) @@ -426,6 +451,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount isKnownTestsEnabled = configurationResponse.libraryConfig?.isKnownTestsEnabled isTestManagementTestsEnabled = configurationResponse.libraryConfig?.isTestManagementEnabled + testManagementAttemptToFixRetries = configurationResponse.libraryConfig?.testManagementAttemptToFixRetries if (isKnownTestsEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) @@ -576,22 +602,25 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } let isNew = false + let isAttemptToFix = false let isDisabled = false let isQuarantined = false - if (isKnownTestsEnabled) { - isNew = isNewTest(testSuitePath, pickle.name) - if (isNew) { - numRetriesByPickleId.set(pickle.id, 0) - } - } if (isTestManagementTestsEnabled) { const testProperties = getTestProperties(testSuitePath, pickle.name) + isAttemptToFix = testProperties.attemptToFix isDisabled = testProperties.disabled - if (isDisabled) { + isQuarantined = testProperties.quarantined + // If attempt to fix is enabled, we run even if the test is disabled + if (!isAttemptToFix && isDisabled) { this.options.dryRun = true - } else { - isQuarantined = testProperties.quarantined + } + } + + if (isKnownTestsEnabled && !isAttemptToFix) { + isNew = isNewTest(testSuitePath, pickle.name) + if (isNew) { + numRetriesByPickleId.set(pickle.id, 0) } } // TODO: for >=11 we could use `runTestCaseResult` instead of accumulating results in `lastStatusByPickleId` @@ -599,6 +628,15 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa const testStatuses = lastStatusByPickleId.get(pickle.id) const lastTestStatus = testStatuses[testStatuses.length - 1] + + // New tests should not be marked as attempt to fix, so EFD + Attempt to fix should not be enabled at the same time + if (isAttemptToFix && lastTestStatus !== 'skip') { + for (let retryIndex = 0; retryIndex < testManagementAttemptToFixRetries; retryIndex++) { + numRetriesByPickleId.set(pickle.id, retryIndex + 1) + runTestCaseResult = await runTestCaseFunction.apply(this, arguments) + } + } + // If it's a new test and it hasn't been skipped, we run it again if (isEarlyFlakeDetectionEnabled && lastTestStatus !== 'skip' && isNew) { for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { @@ -608,7 +646,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } let testStatus = lastTestStatus let shouldBePassedByEFD = false - let shouldBePassedByQuarantine = false + let shouldBePassedByTestManagement = false if (isNew && isEarlyFlakeDetectionEnabled) { /** * If Early Flake Detection (EFD) is enabled the logic is as follows: @@ -625,9 +663,9 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } } - if (isTestManagementTestsEnabled && isQuarantined) { + if (isTestManagementTestsEnabled && (isDisabled || isQuarantined)) { this.success = true - shouldBePassedByQuarantine = true + shouldBePassedByTestManagement = true } if (!pickleResultByFile[testFileAbsolutePath]) { @@ -661,8 +699,8 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa return shouldBePassedByEFD } - if (isNewerCucumberVersion && isTestManagementTestsEnabled && isQuarantined) { - return shouldBePassedByQuarantine + if (isNewerCucumberVersion && isTestManagementTestsEnabled && (isQuarantined || isDisabled)) { + return shouldBePassedByTestManagement } return runTestCaseResult diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index 1f16f98fe73..e53251aa3d9 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -13,7 +13,9 @@ const { addEfdStringToTestName, removeEfdStringFromTestName, getIsFaultyEarlyFlakeDetection, - JEST_WORKER_LOGS_PAYLOAD_CODE + JEST_WORKER_LOGS_PAYLOAD_CODE, + addAttemptToFixStringToTestName, + removeAttemptToFixStringFromTestName } = require('../../dd-trace/src/plugins/util/test') const { getFormattedJestTestParameters, @@ -73,6 +75,7 @@ let hasFilteredSkippableSuites = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false let testManagementTests = {} +let testManagementAttemptToFixRetries = 0 const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -80,6 +83,7 @@ const asyncResources = new WeakMap() const originalTestFns = new WeakMap() const retriedTestsToNumAttempts = new Map() const newTestsTestStatuses = new Map() +const attemptToFixRetriedTestsStatuses = new Map() const BREAKPOINT_HIT_GRACE_PERIOD_MS = 200 @@ -110,7 +114,7 @@ function getTestEnvironmentOptions (config) { return {} } -function getEfdStats (testStatuses) { +function getTestStats (testStatuses) { return testStatuses.reduce((acc, testStatus) => { acc[testStatus]++ return acc @@ -169,6 +173,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (this.isTestManagementTestsEnabled) { try { const hasTestManagementTests = !!testManagementTests.jest + testManagementAttemptToFixRetries = this.testEnvironmentOptions._ddTestManagementAttemptToFixRetries this.testManagementTestsForThisSuite = hasTestManagementTests ? this.getTestManagementTestsForSuite(testManagementTests.jest.suites?.[this.testSuite]?.tests) : this.getTestManagementTestsForSuite(this.testEnvironmentOptions._ddTestManagementTests) @@ -213,9 +218,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (this.testManagementTestsForThisSuite) { return this.testManagementTestsForThisSuite } - // TODO - ADD ATTEMPT_TO_FIX tests if (!testManagementTests) { return { + attemptToFix: [], disabled: [], quarantined: [] } @@ -228,14 +233,19 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } const result = { + attemptToFix: [], disabled: [], quarantined: [] } Object.entries(testManagementTestsForSuite).forEach(([testName, { properties }]) => { + if (properties?.attempt_to_fix) { + result.attemptToFix.push(testName) + } if (properties?.disabled) { result.disabled.push(testName) - } else if (properties?.quarantined) { + } + if (properties?.quarantined) { result.quarantined.push(testName) } }) @@ -243,11 +253,30 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { return result } - // Add the `add_test` event we don't have the test object yet, so + // Generic function to handle test retries + retryTest (testName, retryCount, addRetryStringToTestName, retryType, event) { + // Retrying snapshots has proven to be problematic, so we'll skip them for now + // We'll still detect new tests, but we won't retry them. + // TODO: do not bail out of retrying tests for the whole test suite + if (this.getHasSnapshotTests()) { + log.warn(`${retryType} is disabled for suites with snapshots`) + return + } + + for (let retryIndex = 0; retryIndex < retryCount; retryIndex++) { + if (this.global.test) { + this.global.test(addRetryStringToTestName(testName, retryIndex), event.fn, event.timeout) + } else { + log.error(`${retryType} could not retry test because global.test is undefined`) + } + } + } + + // At the `add_test` event we don't have the test object yet, so we can't use it getTestNameFromAddTestEvent (event, state) { const describeSuffix = getJestTestName(state.currentDescribeBlock) const fullTestName = describeSuffix ? `${describeSuffix} ${event.testName}` : event.testName - return removeEfdStringFromTestName(fullTestName) + return removeAttemptToFixStringFromTestName(removeEfdStringFromTestName(fullTestName)) } async handleTestEvent (event, state) { @@ -273,15 +302,31 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (event.name === 'test_start') { let isNewTest = false let numEfdRetry = null + let numOfAttemptsToFixRetries = null const testParameters = getTestParametersString(this.nameToParams, event.test.name) // Async resource for this test is created here // It is used later on by the test_done handler const asyncResource = new AsyncResource('bound-anonymous-fn') asyncResources.set(event.test, asyncResource) const testName = getJestTestName(event.test) + const originalTestName = removeEfdStringFromTestName(removeAttemptToFixStringFromTestName(testName)) + + let isAttemptToFix = false + let isDisabled = false + let isQuarantined = false + if (this.isTestManagementTestsEnabled) { + isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(originalTestName) + isDisabled = this.testManagementTestsForThisSuite?.disabled?.includes(originalTestName) + isQuarantined = this.testManagementTestsForThisSuite?.quarantined?.includes(originalTestName) + if (isAttemptToFix) { + numOfAttemptsToFixRetries = retriedTestsToNumAttempts.get(originalTestName) + retriedTestsToNumAttempts.set(originalTestName, numOfAttemptsToFixRetries + 1) + } else if (isDisabled) { + event.test.mode = 'skip' + } + } if (this.isKnownTestsEnabled) { - const originalTestName = removeEfdStringFromTestName(testName) isNewTest = retriedTestsToNumAttempts.has(originalTestName) if (isNewTest) { numEfdRetry = retriedTestsToNumAttempts.get(originalTestName) @@ -289,16 +334,10 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } - if (this.isTestManagementTestsEnabled) { - const isDisabled = this.testManagementTestsForThisSuite?.disabled?.includes(testName) - if (isDisabled) { - event.test.mode = 'skip' - } - } const isJestRetry = event.test?.invocations > 1 asyncResource.runInAsyncScope(() => { testStartCh.publish({ - name: removeEfdStringFromTestName(testName), + name: originalTestName, suite: this.testSuite, testSourceFile: this.testSourceFile, displayName: this.displayName, @@ -306,34 +345,46 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { frameworkVersion: jestVersion, isNew: isNewTest, isEfdRetry: numEfdRetry > 0, - isJestRetry + isAttemptToFix, + isAttemptToFixRetry: numOfAttemptsToFixRetries > 0, + isJestRetry, + isDisabled, + isQuarantined }) originalTestFns.set(event.test, event.test.fn) event.test.fn = asyncResource.bind(event.test.fn) }) } + if (event.name === 'add_test') { + const originalTestName = this.getTestNameFromAddTestEvent(event, state) + + const isSkipped = event.mode === 'todo' || event.mode === 'skip' + if (this.isTestManagementTestsEnabled) { + const isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(originalTestName) + if (isAttemptToFix && !isSkipped && !retriedTestsToNumAttempts.has(originalTestName)) { + retriedTestsToNumAttempts.set(originalTestName, 0) + this.retryTest( + event.testName, + testManagementAttemptToFixRetries, + addAttemptToFixStringToTestName, + 'Test Management (Attempt to Fix)', + event + ) + } + } if (this.isKnownTestsEnabled) { - const testName = this.getTestNameFromAddTestEvent(event, state) - const isNew = !this.knownTestsForThisSuite?.includes(testName) - const isSkipped = event.mode === 'todo' || event.mode === 'skip' - if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(testName)) { - retriedTestsToNumAttempts.set(testName, 0) + const isNew = !this.knownTestsForThisSuite?.includes(originalTestName) + if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(originalTestName)) { + retriedTestsToNumAttempts.set(originalTestName, 0) if (this.isEarlyFlakeDetectionEnabled) { - // Retrying snapshots has proven to be problematic, so we'll skip them for now - // We'll still detect new tests, but we won't retry them. - // TODO: do not bail out of EFD with the whole test suite - if (this.getHasSnapshotTests()) { - log.warn('Early flake detection is disabled for suites with snapshots') - return - } - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { - if (this.global.test) { - this.global.test(addEfdStringToTestName(event.testName, retryIndex), event.fn, event.timeout) - } else { - log.error('Early flake detection could not retry test because global.test is undefined') - } - } + this.retryTest( + event.testName, + earlyFlakeDetectionNumRetries, + addEfdStringToTestName, + 'Early flake detection', + event + ) } } } @@ -346,6 +397,32 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { // restore in case it is retried event.test.fn = originalTestFns.get(event.test) + let attemptToFixPassed = false + let failedAllTests = false + if (this.isTestManagementTestsEnabled) { + const testName = getJestTestName(event.test) + const originalTestName = removeAttemptToFixStringFromTestName(testName) + const isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(originalTestName) + if (isAttemptToFix) { + if (attemptToFixRetriedTestsStatuses.has(originalTestName)) { + attemptToFixRetriedTestsStatuses.get(originalTestName).push(status) + } else { + attemptToFixRetriedTestsStatuses.set(originalTestName, [status]) + } + const testStatuses = attemptToFixRetriedTestsStatuses.get(originalTestName) + // Check if this is the last attempt to fix. + // If it is, we'll set the failedAllTests flag to true if all the tests failed + // If all tests passed, we'll set the attemptToFixPassed flag to true + if (testStatuses.length === testManagementAttemptToFixRetries + 1) { + if (testStatuses.every(status => status === 'fail')) { + failedAllTests = true + } else if (testStatuses.every(status => status === 'pass')) { + attemptToFixPassed = true + } + } + } + } + // We'll store the test statuses of the retries if (this.isKnownTestsEnabled) { const testName = getJestTestName(event.test) @@ -359,12 +436,6 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } } - let isQuarantined = false - - if (this.isTestManagementTestsEnabled) { - const testName = getJestTestName(event.test) - isQuarantined = this.testManagementTestsForThisSuite?.quarantined?.includes(testName) - } const promises = {} const numRetries = this.global[RETRY_TIMES] @@ -399,7 +470,8 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { testFinishCh.publish({ status, testStartLine: getTestLineStart(event.test.asyncError, this.testSuite), - isQuarantined + attemptToFixPassed, + failedAllTests }) }) @@ -552,6 +624,7 @@ function cliWrapper (cli, jestVersion) { earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (err) { log.error('Jest library configuration error', err) @@ -710,7 +783,7 @@ function cliWrapper (cli, jestVersion) { if (isEarlyFlakeDetectionEnabled) { let numFailedTestsToIgnore = 0 for (const testStatuses of newTestsTestStatuses.values()) { - const { pass, fail } = getEfdStats(testStatuses) + const { pass, fail } = getTestStats(testStatuses) if (pass > 0) { // as long as one passes, we'll consider the test passed numFailedTestsToIgnore += fail } @@ -725,29 +798,41 @@ function cliWrapper (cli, jestVersion) { const failedTests = result .results .testResults.flatMap(({ testResults, testFilePath: testSuiteAbsolutePath }) => ( - testResults.map(({ fullName: testName, status }) => ({ testName, testSuiteAbsolutePath, status })) + testResults.map(({ fullName: testName, status }) => ( + { testName, testSuiteAbsolutePath, status } + )) )) .filter(({ status }) => status === 'failed') let numFailedQuarantinedTests = 0 + let numFailedQuarantinedOrDisabledAttemptedToFixTests = 0 for (const { testName, testSuiteAbsolutePath } of failedTests) { const testSuite = getTestSuitePath(testSuiteAbsolutePath, result.globalConfig.rootDir) - const isQuarantined = testManagementTests + const originalName = removeAttemptToFixStringFromTestName(testName) + const testManagementTest = testManagementTests ?.jest ?.suites ?.[testSuite] ?.tests - ?.[testName] + ?.[originalName] ?.properties - ?.quarantined - if (isQuarantined) { + // This uses `attempt_to_fix` because this is always the main process and it's not formatted in camelCase + if (testManagementTest?.attempt_to_fix && (testManagementTest?.quarantined || testManagementTest?.disabled)) { + numFailedQuarantinedOrDisabledAttemptedToFixTests++ + } else if (testManagementTest?.quarantined) { numFailedQuarantinedTests++ } } // If every test that failed was quarantined, we'll consider the suite passed - if (numFailedQuarantinedTests !== 0 && result.results.numFailedTests === numFailedQuarantinedTests) { + // Note that if a test is attempted to fix, + // it's considered quarantined both if it's disabled and if it's quarantined (it'll run but its status is ignored) + if ( + (numFailedQuarantinedOrDisabledAttemptedToFixTests !== 0 || numFailedQuarantinedTests !== 0) && + result.results.numFailedTests === + numFailedQuarantinedTests + numFailedQuarantinedOrDisabledAttemptedToFixTests + ) { result.results.success = true } } @@ -947,6 +1032,7 @@ addHook({ _ddIsKnownTestsEnabled, _ddIsTestManagementTestsEnabled, _ddTestManagementTests, + _ddTestManagementAttemptToFixRetries, ...restOfTestEnvironmentOptions } = testEnvironmentOptions diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index 6589e3ccacb..70427ac05dc 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -30,7 +30,9 @@ const { newTests, testsQuarantined, getTestFullName, - getRunTestsWrapper + getRunTestsWrapper, + testsAttemptToFix, + testsStatuses } = require('./utils') require('./common') @@ -138,16 +140,26 @@ function getOnEndHandler (isParallel) { } } + // We substract the errors of attempt to fix tests (quarantined or disabled) from the total number of failures // We subtract the errors from quarantined tests from the total number of failures if (config.isTestManagementTestsEnabled) { let numFailedQuarantinedTests = 0 + let numFailedRetriedQuarantinedOrDisabledTests = 0 + for (const test of testsAttemptToFix) { + const testName = getTestFullName(test) + const testProperties = getTestProperties(test, config.testManagementTests) + if (isTestFailed(test) && (testProperties.isQuarantined || testProperties.isDisabled)) { + const numFailedTests = testsStatuses.get(testName).filter(status => status === 'fail').length + numFailedRetriedQuarantinedOrDisabledTests += numFailedTests + } + } for (const test of testsQuarantined) { if (isTestFailed(test)) { numFailedQuarantinedTests++ } } - this.stats.failures -= numFailedQuarantinedTests - this.failures -= numFailedQuarantinedTests + this.stats.failures -= numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests + this.failures -= numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests } if (status === 'fail') { @@ -193,6 +205,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { if (err) { config.testManagementTests = {} config.isTestManagementTestsEnabled = false + config.testManagementAttemptToFixRetries = 0 } else { config.testManagementTests = receivedTestManagementTests } @@ -260,6 +273,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold config.isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled config.isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + config.testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries // ITR and auto test retries are not supported in parallel mode yet config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled @@ -401,7 +415,7 @@ addHook({ this.on('test', getOnTestHandler(true)) - this.on('test end', getOnTestEndHandler()) + this.on('test end', getOnTestEndHandler(config)) this.on('retry', getOnTestRetryHandler()) @@ -637,6 +651,8 @@ addHook({ if (config.isTestManagementTestsEnabled) { const testSuiteTestManagementTests = config.testManagementTests?.mocha?.suites?.[testPath] || {} newWorkerArgs._ddIsTestManagementTestsEnabled = true + // TODO: attempt to fix does not work in parallel mode yet + // newWorkerArgs._ddTestManagementAttemptToFixRetries = config.testManagementAttemptToFixRetries newWorkerArgs._ddTestManagementTests = { mocha: { suites: { diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index ce33d1cf7c4..27f8d4faf4b 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -3,7 +3,9 @@ const { getTestSuitePath, removeEfdStringFromTestName, - addEfdStringToTestName + addEfdStringToTestName, + addAttemptToFixStringToTestName, + removeAttemptToFixStringFromTestName } = require('../../../dd-trace/src/plugins/util/test') const { channel, AsyncResource } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') @@ -26,7 +28,9 @@ const testToStartLine = new WeakMap() const testFileToSuiteAr = new Map() const wrappedFunctions = new WeakSet() const newTests = {} +const testsAttemptToFix = new Set() const testsQuarantined = new Set() +const testsStatuses = new Map() function getAfterEachHooks (testOrHook) { const hooks = [] @@ -44,10 +48,10 @@ function getTestProperties (test, testManagementTests) { const testSuite = getTestSuitePath(test.file, process.cwd()) const testName = test.fullTitle() - const { disabled: isDisabled, quarantined: isQuarantined } = + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = testManagementTests?.mocha?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } function isNewTest (test, knownTests) { @@ -57,15 +61,18 @@ function isNewTest (test, knownTests) { return !testsForSuite.includes(testName) } -function retryTest (test, earlyFlakeDetectionNumRetries) { +function retryTest (test, numRetries, modifyTestName, tags) { const originalTestName = test.title const suite = test.parent - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + for (let retryIndex = 0; retryIndex < numRetries; retryIndex++) { const clonedTest = test.clone() - clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) + clonedTest.title = modifyTestName(originalTestName, retryIndex + 1) suite.addTest(clonedTest) - clonedTest._ddIsNew = true - clonedTest._ddIsEfdRetry = true + tags.forEach(tag => { + if (tag) { + clonedTest[tag] = true + } + }) } } @@ -102,7 +109,10 @@ function getIsLastRetry (test) { } function getTestFullName (test) { - return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` + const testName = removeEfdStringFromTestName( + removeAttemptToFixStringFromTestName(test.fullTitle()) + ) + return `mocha.${getTestSuitePath(test.file, process.cwd())}.${testName}` } function getTestStatus (test) { @@ -195,12 +205,15 @@ function getOnTestHandler (isMain) { title, _ddIsNew: isNew, _ddIsEfdRetry: isEfdRetry, + _ddIsAttemptToFix: isAttemptToFix, _ddIsDisabled: isDisabled, _ddIsQuarantined: isQuarantined } = test + const testName = removeEfdStringFromTestName(removeAttemptToFixStringFromTestName(test.fullTitle())) + const testInfo = { - testName: test.fullTitle(), + testName, testSuiteAbsolutePath, title, testStartLine @@ -212,6 +225,7 @@ function getOnTestHandler (isMain) { testInfo.isNew = isNew testInfo.isEfdRetry = isEfdRetry + testInfo.isAttemptToFix = isAttemptToFix testInfo.isDisabled = isDisabled testInfo.isQuarantined = isQuarantined // We want to store the result of the new tests @@ -224,7 +238,7 @@ function getOnTestHandler (isMain) { } } - if (isDisabled) { + if (!isAttemptToFix && isDisabled) { test.pending = true } @@ -234,7 +248,7 @@ function getOnTestHandler (isMain) { } } -function getOnTestEndHandler () { +function getOnTestEndHandler (config) { return async function (test) { const asyncResource = getTestAsyncResource(test) const status = getTestStatus(test) @@ -249,13 +263,40 @@ function getOnTestEndHandler () { }) } + let hasFailedAllRetries = false + let attemptToFixPassed = false + + const testName = getTestFullName(test) + + if (!testsStatuses.get(testName)) { + testsStatuses.set(testName, [status]) + } else { + testsStatuses.get(testName).push(status) + } + const testStatuses = testsStatuses.get(testName) + + const isLastAttempt = testStatuses.length === config.testManagementAttemptToFixRetries + 1 + + if (test._ddIsAttemptToFix && isLastAttempt) { + if (testStatuses.every(status => status === 'fail')) { + hasFailedAllRetries = true + } else if (testStatuses.every(status => status === 'pass')) { + attemptToFixPassed = true + } + } + + const isAttemptToFixRetry = test._ddIsAttemptToFix && testStatuses.length > 1 + // if there are afterEach to be run, we don't finish the test yet if (asyncResource && !getAfterEachHooks(test).length) { asyncResource.runInAsyncScope(() => { testFinishCh.publish({ status, hasBeenRetried: isMochaRetry(test), - isLastRetry: getIsLastRetry(test) + isLastRetry: getIsLastRetry(test), + hasFailedAllRetries, + attemptToFixPassed, + isAttemptToFixRetry }) }) } @@ -374,33 +415,50 @@ function getOnPendingHandler () { } } -// Hook to add retries to tests if EFD is enabled +// Hook to add retries to tests if Test Management or EFD is enabled function getRunTestsWrapper (runTests, config) { - return function (suite, fn) { + return function (suite) { + if (config.isTestManagementTestsEnabled) { + suite.tests.forEach((test) => { + const { isAttemptToFix, isDisabled, isQuarantined } = getTestProperties(test, config.testManagementTests) + if (isAttemptToFix && !test.isPending()) { + test._ddIsAttemptToFix = true + test._ddIsDisabled = isDisabled + test._ddIsQuarantined = isQuarantined + // This is needed to know afterwards which ones have been retried to ignore its result + testsAttemptToFix.add(test) + retryTest( + test, + config.testManagementAttemptToFixRetries, + addAttemptToFixStringToTestName, + ['_ddIsAttemptToFix', isDisabled && '_ddIsDisabled', isQuarantined && '_ddIsQuarantined'] + ) + } else if (isDisabled) { + test._ddIsDisabled = true + } else if (isQuarantined) { + testsQuarantined.add(test) + test._ddIsQuarantined = true + } + }) + } + if (config.isKnownTestsEnabled) { // by the time we reach `this.on('test')`, it is too late. We need to add retries here suite.tests.forEach(test => { if (!test.isPending() && isNewTest(test, config.knownTests)) { test._ddIsNew = true if (config.isEarlyFlakeDetectionEnabled) { - retryTest(test, config.earlyFlakeDetectionNumRetries) + retryTest( + test, + config.earlyFlakeDetectionNumRetries, + addEfdStringToTestName, + ['_ddIsNew', '_ddIsEfdRetry'] + ) } } }) } - if (config.isTestManagementTestsEnabled) { - suite.tests.forEach(test => { - const { isDisabled, isQuarantined } = getTestProperties(test, config.testManagementTests) - if (isDisabled) { - test._ddIsDisabled = true - } else if (isQuarantined) { - testsQuarantined.add(test) - test._ddIsQuarantined = true - } - }) - } - return runTests.apply(this, arguments) } } @@ -408,7 +466,6 @@ function getRunTestsWrapper (runTests, config) { module.exports = { isNewTest, getTestProperties, - retryTest, getSuitesByTestFile, isMochaRetry, getTestFullName, @@ -427,5 +484,7 @@ module.exports = { testFileToSuiteAr, getRunTestsWrapper, newTests, - testsQuarantined + testsQuarantined, + testsAttemptToFix, + testsStatuses } diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index d456150036f..c8f58109f0e 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -36,6 +36,8 @@ addHook({ } if (this.options._ddIsTestManagementTestsEnabled) { config.isTestManagementTestsEnabled = true + // TODO: attempt to fix does not work in parallel mode yet + // config.testManagementAttemptToFixRetries = this.options._ddTestManagementAttemptToFixRetries config.testManagementTests = this.options._ddTestManagementTests delete this.options._ddIsTestManagementTestsEnabled delete this.options._ddTestManagementTests @@ -64,7 +66,7 @@ addHook({ }) this.on('test', getOnTestHandler(false)) - this.on('test end', getOnTestEndHandler()) + this.on('test end', getOnTestEndHandler(config)) // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted this.on('hook end', getOnHookEndHandler()) diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index f56f88a565f..0df5addc442 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -22,6 +22,7 @@ const testToAr = new WeakMap() const testSuiteToAr = new Map() const testSuiteToTestStatuses = new Map() const testSuiteToErrors = new Map() +const testsToTestStatuses = new Map() const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') let applyRepeatEachIndex = null @@ -43,7 +44,9 @@ let isFlakyTestRetriesEnabled = false let flakyTestRetriesCount = 0 let knownTests = {} let isTestManagementTestsEnabled = false +let testManagementAttemptToFixRetries = 0 let testManagementTests = {} +const quarantinedOrDisabledTestsAttemptToFix = [] let rootDir = '' const MINIMUM_SUPPORTED_VERSION_RANGE_EFD = '>=1.38.0' @@ -51,10 +54,9 @@ function getTestProperties (test) { const testName = getTestFullname(test) const testSuite = getTestSuitePath(test._requireFile, rootDir) - const { disabled, quarantined } = + const { attempt_to_fix: attemptToFix, disabled, quarantined } = testManagementTests?.playwright?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - - return { disabled, quarantined } + return { attemptToFix, disabled, quarantined } } function isNewTest (test) { @@ -73,16 +75,19 @@ function getSuiteType (test, type) { } // Copy of Suite#_deepClone but with a function to filter tests -function deepCloneSuite (suite, filterTest) { +function deepCloneSuite (suite, filterTest, tags = []) { const copy = suite._clone() for (const entry of suite._entries) { if (entry.constructor.name === 'Suite') { - copy._addSuite(deepCloneSuite(entry, filterTest)) + copy._addSuite(deepCloneSuite(entry, filterTest, tags)) } else { if (filterTest(entry)) { const copiedTest = entry._clone() - copiedTest._ddIsNew = true - copiedTest._ddIsEfdRetry = true + tags.forEach(tag => { + if (tag) { + copiedTest[tag] = true + } + }) copy._addTest(copiedTest) } } @@ -276,6 +281,11 @@ function testBeginHandler (test, browserName) { }) } + // We disable retries by default if attemptToFix is true + if (getTestProperties(test).attemptToFix) { + test.retries = 0 + } + const testAsyncResource = new AsyncResource('bound-anonymous-fn') testToAr.set(test, testAsyncResource) testAsyncResource.runInAsyncScope(() => { @@ -288,7 +298,6 @@ function testBeginHandler (test, browserName) { }) }) } - function testEndHandler (test, annotations, testStatus, error, isTimeout) { let annotationTags if (annotations.length) { @@ -305,6 +314,27 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { return } + const testFullName = getTestFullname(test) + const testFqn = `${testSuiteAbsolutePath} ${testFullName}` + const testStatuses = testsToTestStatuses.get(testFqn) || [] + + if (testStatuses.length === 0) { + testsToTestStatuses.set(testFqn, [testStatus]) + } else { + testStatuses.push(testStatus) + } + + let hasFailedAllRetries = false + let hasPassedAttemptToFixRetries = false + + if (testStatuses.length === testManagementAttemptToFixRetries + 1) { + if (testStatuses.every(status => status === 'fail')) { + hasFailedAllRetries = true + } else if (testStatuses.every(status => status === 'pass')) { + hasPassedAttemptToFixRetries = true + } + } + const testResult = results[results.length - 1] const testAsyncResource = testToAr.get(test) testAsyncResource.runInAsyncScope(() => { @@ -315,8 +345,12 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { error, extraTags: annotationTags, isNew: test._ddIsNew, + isAttemptToFix: test._ddIsAttemptToFix, + isAttemptToFixRetry: test._ddIsAttemptToFixRetry, isQuarantined: test._ddIsQuarantined, - isEfdRetry: test._ddIsEfdRetry + isEfdRetry: test._ddIsEfdRetry, + hasFailedAllRetries, + hasPassedAttemptToFixRetries }) }) @@ -338,7 +372,6 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { // Last test, we finish the suite if (!remainingTestsByFile[testSuiteAbsolutePath].length) { const testStatuses = testSuiteToTestStatuses.get(testSuiteAbsolutePath) - let testSuiteStatus = 'pass' if (testStatuses.some(status => status === 'fail')) { testSuiteStatus = 'fail' @@ -445,6 +478,7 @@ function runnerHook (runnerExport, playwrightVersion) { isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (e) { isEarlyFlakeDetectionEnabled = false @@ -485,7 +519,10 @@ function runnerHook (runnerExport, playwrightVersion) { const projects = getProjectsFromRunner(this) - if (isFlakyTestRetriesEnabled && flakyTestRetriesCount > 0) { + const shouldSetRetries = isFlakyTestRetriesEnabled && + flakyTestRetriesCount > 0 && + !isTestManagementTestsEnabled + if (shouldSetRetries) { projects.forEach(project => { if (project.retries === 0) { // Only if it hasn't been set by the user project.retries = flakyTestRetriesCount @@ -493,7 +530,7 @@ function runnerHook (runnerExport, playwrightVersion) { }) } - const runAllTestsReturn = await runAllTests.apply(this, arguments) + let runAllTestsReturn = await runAllTests.apply(this, arguments) Object.values(remainingTestsByFile).forEach(tests => { // `tests` should normally be empty, but if it isn't, @@ -508,6 +545,26 @@ function runnerHook (runnerExport, playwrightVersion) { const sessionStatus = runAllTestsReturn.status || runAllTestsReturn + if (isTestManagementTestsEnabled && sessionStatus === 'failed') { + let totalFailedTestCount = 0 + let totalAttemptToFixFailedTestCount = 0 + + for (const testStatuses of testsToTestStatuses.values()) { + totalFailedTestCount += testStatuses.filter(status => status === 'fail').length + } + + for (const test of quarantinedOrDisabledTestsAttemptToFix) { + const fullname = getTestFullname(test) + const fqn = `${test._requireFile} ${fullname}` + const testStatuses = testsToTestStatuses.get(fqn) + totalAttemptToFixFailedTestCount += testStatuses.filter(status => status === 'fail').length + } + + if (totalFailedTestCount === totalAttemptToFixFailedTestCount) { + runAllTestsReturn = 'passed' + } + } + const flushWait = new Promise(resolve => { onDone = resolve }) @@ -608,9 +665,28 @@ addHook({ const testProperties = getTestProperties(test) if (testProperties.disabled) { test._ddIsDisabled = true - test.expectedStatus = 'skipped' } else if (testProperties.quarantined) { test._ddIsQuarantined = true + } + if (testProperties.attemptToFix) { + test._ddIsAttemptToFix = true + const fileSuite = getSuiteType(test, 'file') + const projectSuite = getSuiteType(test, 'project') + const isAttemptToFix = test => getTestProperties(test).attemptToFix + for (let repeatEachIndex = 1; repeatEachIndex <= testManagementAttemptToFixRetries; repeatEachIndex++) { + const copyFileSuite = deepCloneSuite(fileSuite, isAttemptToFix, [ + testProperties.disabled && '_ddIsDisabled', + testProperties.quarantined && '_ddIsQuarantined', + '_ddIsAttemptToFix', + '_ddIsAttemptToFixRetry' + ]) + applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) + projectSuite._addSuite(copyFileSuite) + } + if (testProperties.disabled || testProperties.quarantined) { + quarantinedOrDisabledTestsAttemptToFix.push(test) + } + } else if (testProperties.disabled || testProperties.quarantined) { test.expectedStatus = 'skipped' } } @@ -619,18 +695,22 @@ addHook({ if (isKnownTestsEnabled) { const newTests = allTests.filter(isNewTest) - newTests.forEach(newTest => { + for (const newTest of newTests) { + // No need to filter out attempt to fix tests here because attempt to fix tests are never new newTest._ddIsNew = true if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped') { const fileSuite = getSuiteType(newTest, 'file') const projectSuite = getSuiteType(newTest, 'project') - for (let repeatEachIndex = 0; repeatEachIndex < earlyFlakeDetectionNumRetries; repeatEachIndex++) { - const copyFileSuite = deepCloneSuite(fileSuite, isNewTest) + for (let repeatEachIndex = 1; repeatEachIndex <= earlyFlakeDetectionNumRetries; repeatEachIndex++) { + const copyFileSuite = deepCloneSuite(fileSuite, isNewTest, [ + '_ddIsNew', + '_ddIsEfdRetry' + ]) applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) projectSuite._addSuite(copyFileSuite) } } - }) + } } return rootSuite diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index 4ac4062b46c..b3bf9809c4c 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -9,6 +9,7 @@ const testPassCh = channel('ci:vitest:test:pass') const testErrorCh = channel('ci:vitest:test:error') const testSkipCh = channel('ci:vitest:test:skip') const isNewTestCh = channel('ci:vitest:test:is-new') +const isAttemptToFixCh = channel('ci:vitest:test:is-attempt-to-fix') const isDisabledCh = channel('ci:vitest:test:is-disabled') const isQuarantinedCh = channel('ci:vitest:test:is-quarantined') @@ -30,7 +31,9 @@ const taskToStatuses = new WeakMap() const newTasks = new WeakSet() const disabledTasks = new WeakSet() const quarantinedTasks = new WeakSet() +const attemptToFixTasks = new WeakSet() let isRetryReasonEfd = false +let isRetryReasonAttemptToFix = false const switchedStatuses = new WeakSet() const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -53,6 +56,7 @@ function getProvidedContext () { _ddEarlyFlakeDetectionNumRetries: numRepeats, _ddIsKnownTestsEnabled: isKnownTestsEnabled, _ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled, + _ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries, _ddTestManagementTests: testManagementTests, _ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled } = globalThis.__vitest_worker__.providedContext @@ -64,6 +68,7 @@ function getProvidedContext () { numRepeats, isKnownTestsEnabled, isTestManagementTestsEnabled, + testManagementAttemptToFixRetries, testManagementTests, isFlakyTestRetriesEnabled } @@ -76,6 +81,7 @@ function getProvidedContext () { numRepeats: 0, isKnownTestsEnabled: false, isTestManagementTestsEnabled: false, + testManagementAttemptToFixRetries: 0, testManagementTests: {} } } @@ -176,6 +182,7 @@ function getSortWrapper (sort) { let isEarlyFlakeDetectionFaulty = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false + let testManagementAttemptToFixRetries = 0 let isDiEnabled = false let knownTests = {} let testManagementTests = {} @@ -190,6 +197,7 @@ function getSortWrapper (sort) { isDiEnabled = libraryConfig.isDiEnabled isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (e) { isFlakyTestRetriesEnabled = false @@ -262,6 +270,7 @@ function getSortWrapper (sort) { try { const workspaceProject = this.ctx.getCoreWorkspaceProject() workspaceProject._provided._ddIsTestManagementTestsEnabled = isTestManagementTestsEnabled + workspaceProject._provided._ddTestManagementAttemptToFixRetries = testManagementAttemptToFixRetries workspaceProject._provided._ddTestManagementTests = testManagementTests } catch (e) { log.warn('Could not send test management tests to workers so Test Management will not work.') @@ -353,10 +362,24 @@ addHook({ isKnownTestsEnabled, numRepeats, isTestManagementTestsEnabled, + testManagementAttemptToFixRetries, testManagementTests } = getProvidedContext() if (isTestManagementTestsEnabled) { + isAttemptToFixCh.publish({ + testManagementTests, + testSuiteAbsolutePath: task.file.filepath, + testName, + onDone: (isAttemptToFix) => { + if (isAttemptToFix) { + isRetryReasonAttemptToFix = task.repeats !== testManagementAttemptToFixRetries + task.repeats = testManagementAttemptToFixRetries + attemptToFixTasks.add(task) + taskToStatuses.set(task, []) + } + } + }) isDisabledCh.publish({ testManagementTests, testSuiteAbsolutePath: task.file.filepath, @@ -364,7 +387,10 @@ addHook({ onDone: (isTestDisabled) => { if (isTestDisabled) { disabledTasks.add(task) - task.mode = 'skip' + if (!attemptToFixTasks.has(task)) { + // we only actually skip if the test is not being attempted to be fixed + task.mode = 'skip' + } } } }) @@ -376,7 +402,7 @@ addHook({ testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isNew) => { - if (isNew) { + if (isNew && !attemptToFixTasks.has(task)) { if (isEarlyFlakeDetectionEnabled) { isRetryReasonEfd = task.repeats !== numRepeats task.repeats = numRepeats @@ -396,19 +422,28 @@ addHook({ shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => function (task) { const { isEarlyFlakeDetectionEnabled, isTestManagementTestsEnabled } = getProvidedContext() - if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task)) { - const statuses = taskToStatuses.get(task) - // If the test has passed at least once, we consider it passed - if (statuses.includes('pass')) { + if (isTestManagementTestsEnabled) { + const isAttemptingToFix = attemptToFixTasks.has(task) + const isDisabled = disabledTasks.has(task) + const isQuarantined = quarantinedTasks.has(task) + + if (isAttemptingToFix && (isDisabled || isQuarantined)) { if (task.result.state === 'fail') { switchedStatuses.add(task) } task.result.state = 'pass' + } else if (isQuarantined) { + task.result.state = 'pass' } } - if (isTestManagementTestsEnabled) { - if (quarantinedTasks.has(task)) { + if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task)) { + const statuses = taskToStatuses.get(task) + // If the test has passed at least once, we consider it passed + if (statuses.includes('pass')) { + if (task.result.state === 'fail') { + switchedStatuses.add(task) + } task.result.state = 'pass' } } @@ -481,6 +516,8 @@ addHook({ } const lastExecutionStatus = task.result.state + const shouldFlipStatus = isEarlyFlakeDetectionEnabled || attemptToFixTasks.has(task) + const statuses = taskToStatuses.get(task) // These clauses handle task.repeats, whether EFD is enabled or not // The only thing that EFD does is to forcefully pass the test if it has passed at least once @@ -501,15 +538,21 @@ addHook({ testPassCh.publish({ task }) }) } - if (isEarlyFlakeDetectionEnabled) { - const statuses = taskToStatuses.get(task) + if (shouldFlipStatus) { statuses.push(lastExecutionStatus) // If we don't "reset" the result.state to "pass", once a repetition fails, // vitest will always consider the test as failed, so we can't read the actual status + // This means that we change vitest's behavior: + // if the last attempt passes, vitest would consider the test as failed + // but after this change, it will consider the test as passed task.result.state = 'pass' } } } else if (numRepetition === task.repeats) { + if (shouldFlipStatus) { + statuses.push(lastExecutionStatus) + } + const asyncResource = taskToAsync.get(task) if (lastExecutionStatus === 'fail') { const testError = task.result?.errors?.[0] @@ -532,8 +575,11 @@ addHook({ testSuiteAbsolutePath: task.file.filepath, isRetry: numAttempt > 0 || numRepetition > 0, isRetryReasonEfd, + isRetryReasonAttemptToFix: isRetryReasonAttemptToFix && numRepetition > 0, isNew, mightHitProbe: isDiEnabled && numAttempt > 0, + isAttemptToFix: attemptToFixTasks.has(task), + isDisabled: disabledTasks.has(task), isQuarantined }) }) @@ -548,6 +594,8 @@ addHook({ } const result = await onAfterTryTask.apply(this, arguments) + const { testManagementAttemptToFixRetries } = getProvidedContext() + const status = getVitestTestStatus(task, retryCount) const asyncResource = taskToAsync.get(task) @@ -557,10 +605,18 @@ addHook({ await waitForHitProbe() } + let attemptToFixPassed = false + if (attemptToFixTasks.has(task)) { + const statuses = taskToStatuses.get(task) + if (statuses.length === testManagementAttemptToFixRetries && statuses.every(status => status === 'pass')) { + attemptToFixPassed = true + } + } + if (asyncResource) { // We don't finish here because the test might fail in a later hook (afterEach) asyncResource.runInAsyncScope(() => { - testFinishTimeCh.publish({ status, task }) + testFinishTimeCh.publish({ status, task, attemptToFixPassed }) }) } @@ -715,11 +771,19 @@ addHook({ testError = errors[0] } + let hasFailedAllRetries = false + if (attemptToFixTasks.has(task)) { + const statuses = taskToStatuses.get(task) + if (statuses.every(status => status === 'fail')) { + hasFailedAllRetries = true + } + } + if (testAsyncResource) { const isRetry = task.result?.retryCount > 0 // `duration` is the duration of all the retries, so it can't be used if there are retries testAsyncResource.runInAsyncScope(() => { - testErrorCh.publish({ duration: !isRetry ? duration : undefined, error: testError }) + testErrorCh.publish({ duration: !isRetry ? duration : undefined, error: testError, hasFailedAllRetries }) }) } if (errors?.length) { diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index f7ff7e4c2e9..37d58e4fcec 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -30,7 +30,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -328,6 +331,10 @@ class CucumberPlugin extends CiPlugin { isNew, isEfdRetry, isFlakyRetry, + isAttemptToFix, + isAttemptToFixRetry, + hasFailedAllRetries, + hasPassedAllRetries, isDisabled, isQuarantined }) => { @@ -358,6 +365,22 @@ class CucumberPlugin extends CiPlugin { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + + if (isAttemptToFix) { + span.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + } + + if (isAttemptToFixRetry) { + span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + if (hasPassedAllRetries) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + } + if (isDisabled) { span.setTag(TEST_MANAGEMENT_IS_DISABLED, 'true') } diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 8767b6b9016..092db83a901 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -39,7 +39,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, - DD_CAPABILITIES_TEST_IMPACT_ANALYSIS + DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') @@ -64,7 +67,8 @@ const { GIT_COMMIT_SHA, GIT_BRANCH, CI_PROVIDER_NAME, - CI_WORKSPACE_PATH + CI_WORKSPACE_PATH, + GIT_COMMIT_MESSAGE } = require('../../dd-trace/src/plugins/util/tags') const { OS_VERSION, @@ -201,7 +205,8 @@ class CypressPlugin { [RUNTIME_VERSION]: runtimeVersion, [GIT_BRANCH]: branch, [CI_PROVIDER_NAME]: ciProviderName, - [CI_WORKSPACE_PATH]: repositoryRoot + [CI_WORKSPACE_PATH]: repositoryRoot, + [GIT_COMMIT_MESSAGE]: commitMessage } = this.testEnvironmentMetadata this.repositoryRoot = repositoryRoot @@ -217,9 +222,11 @@ class CypressPlugin { runtimeName, runtimeVersion, branch, - testLevel: 'test' + testLevel: 'test', + commitMessage } this.finishedTestsByFile = {} + this.testStatuses = {} this.isTestsSkipped = false this.isSuitesSkippingEnabled = false @@ -233,6 +240,8 @@ class CypressPlugin { this.hasUnskippableSuites = false this.unskippableSuites = [] this.knownTests = [] + this.isTestManagementTestsEnabled = false + this.testManagementAttemptToFixRetries = 0 } // Init function returns a promise that resolves with the Cypress configuration @@ -261,7 +270,8 @@ class CypressPlugin { isFlakyTestRetriesEnabled, flakyTestRetriesCount, isKnownTestsEnabled, - isTestManagementEnabled + isTestManagementEnabled, + testManagementAttemptToFixRetries } } = libraryConfigurationResponse this.isSuitesSkippingEnabled = isSuitesSkippingEnabled @@ -273,17 +283,22 @@ class CypressPlugin { this.cypressConfig.retries.runMode = flakyTestRetriesCount } this.isTestManagementTestsEnabled = isTestManagementEnabled + this.testManagementAttemptToFixRetries = testManagementAttemptToFixRetries } return this.cypressConfig }) return this.libraryConfigurationPromise } + getTestSuiteProperties (testSuite) { + return this.testManagementTests?.cypress?.suites?.[testSuite]?.tests || {} + } + getTestProperties (testSuite, testName) { - const { disabled: isDisabled, quarantined: isQuarantined } = - this.testManagementTests?.cypress?.suites?.[testSuite]?.tests?.[testName]?.properties || {} + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = + this.getTestSuiteProperties(testSuite)?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } getTestSuiteSpan ({ testSuite, testSuiteAbsolutePath }) { @@ -312,7 +327,7 @@ class CypressPlugin { }) } - getTestSpan ({ testName, testSuite, isUnskippable, isForcedToRun, testSourceFile }) { + getTestSpan ({ testName, testSuite, isUnskippable, isForcedToRun, testSourceFile, isDisabled, isQuarantined }) { const testSuiteTags = { [TEST_COMMAND]: this.command, [TEST_COMMAND]: this.command, @@ -357,6 +372,14 @@ class CypressPlugin { testSpanMetadata[TEST_ITR_FORCED_RUN] = 'true' } + if (isDisabled) { + testSpanMetadata[TEST_MANAGEMENT_IS_DISABLED] = 'true' + } + + if (isQuarantined) { + testSpanMetadata[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' + } + this.ciVisEvent(TELEMETRY_EVENT_CREATED, 'test', { hasCodeOwners: !!codeOwners }) return this.tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test`, { @@ -691,7 +714,10 @@ class CypressPlugin { isEarlyFlakeDetectionEnabled: this.isEarlyFlakeDetectionEnabled, knownTestsForSuite: this.knownTestsByTestSuite?.[testSuite] || [], earlyFlakeDetectionNumRetries: this.earlyFlakeDetectionNumRetries, - isKnownTestsEnabled: this.isKnownTestsEnabled + isKnownTestsEnabled: this.isKnownTestsEnabled, + isTestManagementEnabled: this.isTestManagementTestsEnabled, + testManagementAttemptToFixRetries: this.testManagementAttemptToFixRetries, + testManagementTests: this.getTestSuiteProperties(testSuite) } if (this.testSuiteSpan) { @@ -707,8 +733,7 @@ class CypressPlugin { }) const isUnskippable = this.unskippableSuites.includes(testSuite) const isForcedToRun = shouldSkip && isUnskippable - const { isDisabled, isQuarantined } = this.getTestProperties(testSuite, testName) - + const { isAttemptToFix, isDisabled, isQuarantined } = this.getTestProperties(testSuite, testName) // skip test if (shouldSkip && !isUnskippable) { this.skippedTests.push(test) @@ -718,7 +743,7 @@ class CypressPlugin { // TODO: I haven't found a way to trick cypress into ignoring a test // The way we'll implement quarantine in cypress is by skipping the test altogether - if (isDisabled || isQuarantined) { + if (!isAttemptToFix && (isDisabled || isQuarantined)) { return { shouldSkip: true } } @@ -727,7 +752,9 @@ class CypressPlugin { testName, testSuite, isUnskippable, - isForcedToRun + isForcedToRun, + isDisabled, + isQuarantined }) } @@ -747,7 +774,8 @@ class CypressPlugin { testSuiteAbsolutePath, testName, isNew, - isEfdRetry + isEfdRetry, + isAttemptToFix } = test if (coverage && this.isCodeCoverageEnabled && this.tracer._tracer._exporter?.exportCoverage) { const coverageFiles = getCoveredFilenamesFromCoverage(coverage) @@ -770,6 +798,14 @@ class CypressPlugin { const testStatus = CYPRESS_STATUS_TO_TEST_STATUS[state] this.activeTestSpan.setTag(TEST_STATUS, testStatus) + // Save the test status to know if it has passed all retries + if (!this.testStatuses[testName]) { + this.testStatuses[testName] = [testStatus] + } else { + this.testStatuses[testName].push(testStatus) + } + const testStatuses = this.testStatuses[testName] + if (error) { this.activeTestSpan.setTag('error', error) } @@ -786,6 +822,22 @@ class CypressPlugin { this.activeTestSpan.setTag(TEST_RETRY_REASON, 'efd') } } + if (isAttemptToFix) { + this.activeTestSpan.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (testStatuses.length > 1) { + this.activeTestSpan.setTag(TEST_IS_RETRY, 'true') + this.activeTestSpan.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + } + const isLastAttempt = testStatuses.length === this.testManagementAttemptToFixRetries + 1 + if (isLastAttempt) { + if (testStatuses.every(status => status === 'fail')) { + this.activeTestSpan.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } else if (testStatuses.every(status => status === 'pass')) { + this.activeTestSpan.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + } + } + const finishedTest = { testName, testStatus, diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index 749a25d7f66..668e146a2f5 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -4,6 +4,9 @@ let isKnownTestsEnabled = false let knownTestsForSuite = [] let suiteTests = [] let earlyFlakeDetectionNumRetries = 0 +let isTestManagementEnabled = false +let testManagementAttemptToFixRetries = 0 +let testManagementTests = {} // We need to grab the original window as soon as possible, // in case the test changes the origin. If the test does change the origin, // any call to `cy.window()` will result in a cross origin error. @@ -23,31 +26,53 @@ function isNewTest (test) { return !knownTestsForSuite.includes(test.fullTitle()) } -function retryTest (test, suiteTests) { - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { +function getTestProperties (testName) { + // We neeed to do it in this way because of compatibility with older versions as '?' is not supported in older versions of Cypress + const properties = testManagementTests[testName] && testManagementTests[testName].properties || {}; + + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = properties; + + return { isAttemptToFix, isDisabled, isQuarantined }; +} + +function retryTest (test, suiteTests, numRetries, tags) { + for (let retryIndex = 0; retryIndex < numRetries; retryIndex++) { const clonedTest = test.clone() // TODO: signal in framework logs that this is a retry. // TODO: Change it so these tests are allowed to fail. // TODO: figure out if reported duration is skewed. suiteTests.unshift(clonedTest) - clonedTest._ddIsNew = true - clonedTest._ddIsEfdRetry = true + tags.forEach(tag => { + clonedTest[tag] = true + }) } } const oldRunTests = Cypress.mocha.getRunner().runTests Cypress.mocha.getRunner().runTests = function (suite, fn) { - if (!isKnownTestsEnabled) { + if (!isKnownTestsEnabled && !isTestManagementEnabled) { return oldRunTests.apply(this, arguments) } // We copy the new tests at the beginning of the suite run (runTests), so that they're run // multiple times. suite.tests.forEach(test => { - if (!test._ddIsNew && !test.isPending() && isNewTest(test)) { - test._ddIsNew = true - if (isEarlyFlakeDetectionEnabled) { - retryTest(test, suite.tests) + const testName = test.fullTitle() + + const { isAttemptToFix } = getTestProperties(testName) + + if (isTestManagementEnabled) { + if (isAttemptToFix && !test.isPending()) { + test._ddIsAttemptToFix = true + retryTest(test, suite.tests, testManagementAttemptToFixRetries, ['_ddIsAttemptToFix']) + } + } + if (isKnownTestsEnabled) { + if (!test._ddIsNew && !test.isPending() && isNewTest(test)) { + test._ddIsNew = true + if (isEarlyFlakeDetectionEnabled && !isAttemptToFix) { + retryTest(test, suite.tests, earlyFlakeDetectionNumRetries, ['_ddIsNew', '_ddIsEfdRetry']) + } } } }) @@ -80,6 +105,9 @@ before(function () { isKnownTestsEnabled = suiteConfig.isKnownTestsEnabled knownTestsForSuite = suiteConfig.knownTestsForSuite earlyFlakeDetectionNumRetries = suiteConfig.earlyFlakeDetectionNumRetries + isTestManagementEnabled = suiteConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = suiteConfig.testManagementAttemptToFixRetries + testManagementTests = suiteConfig.testManagementTests } }) }) @@ -104,7 +132,8 @@ afterEach(function () { state: currentTest.state, error: currentTest.err, isNew: currentTest._ddIsNew, - isEfdRetry: currentTest._ddIsEfdRetry + isEfdRetry: currentTest._ddIsEfdRetry, + isAttemptToFix: currentTest._ddIsAttemptToFix } try { testInfo.testSourceLine = Cypress.mocha.getRunner().currentRunnable.invocationDetails.line diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 1e231443772..e7b36d17341 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -27,7 +27,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const id = require('../../dd-trace/src/id') @@ -177,6 +180,7 @@ class JestPlugin extends CiPlugin { config._ddRepositoryRoot = this.repositoryRoot config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false config._ddIsTestManagementTestsEnabled = this.libraryConfig?.isTestManagementEnabled ?? false + config._ddTestManagementAttemptToFixRetries = this.libraryConfig?.testManagementAttemptToFixRetries ?? 0 config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount config._ddIsDiEnabled = this.libraryConfig?.isDiEnabled ?? false config._ddIsKnownTestsEnabled = this.libraryConfig?.isKnownTestsEnabled ?? false @@ -336,14 +340,22 @@ class JestPlugin extends CiPlugin { this.activeTestSpan = span }) - this.addSub('ci:jest:test:finish', ({ status, testStartLine, isQuarantined }) => { + this.addSub('ci:jest:test:finish', ({ + status, + testStartLine, + attemptToFixPassed, + failedAllTests + }) => { const span = storage('legacy').getStore().span span.setTag(TEST_STATUS, status) if (testStartLine) { span.setTag(TEST_SOURCE_START, testStartLine) } - if (isQuarantined) { - span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + if (failedAllTests) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') } const spanTags = span.context()._tags @@ -407,7 +419,11 @@ class JestPlugin extends CiPlugin { testSourceFile, isNew, isEfdRetry, - isJestRetry + isAttemptToFix, + isAttemptToFixRetry, + isJestRetry, + isDisabled, + isQuarantined } = test const extraTags = { @@ -425,6 +441,23 @@ class JestPlugin extends CiPlugin { extraTags[JEST_DISPLAY_NAME] = displayName } + if (isAttemptToFix) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + } + + if (isAttemptToFixRetry) { + extraTags[TEST_IS_RETRY] = 'true' + extraTags[TEST_RETRY_REASON] = 'attempt_to_fix' + } + + if (isDisabled) { + extraTags[TEST_MANAGEMENT_IS_DISABLED] = 'true' + } + + if (isQuarantined) { + extraTags[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' + } + if (isNew) { extraTags[TEST_IS_NEW] = 'true' if (isEfdRetry) { diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 9df19700292..ef3867804a9 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -17,7 +17,6 @@ const { TEST_CODE_OWNERS, ITR_CORRELATION_ID, TEST_SOURCE_FILE, - removeEfdStringFromTestName, TEST_IS_NEW, TEST_IS_RETRY, TEST_EARLY_FLAKE_ENABLED, @@ -34,7 +33,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -199,7 +201,14 @@ class MochaPlugin extends CiPlugin { this.tracer._exporter.flush() }) - this.addSub('ci:mocha:test:finish', ({ status, hasBeenRetried, isLastRetry }) => { + this.addSub('ci:mocha:test:finish', ({ + status, + hasBeenRetried, + isLastRetry, + hasFailedAllRetries, + attemptToFixPassed, + isAttemptToFixRetry + }) => { const store = storage('legacy').getStore() const span = store?.span @@ -208,6 +217,16 @@ class MochaPlugin extends CiPlugin { if (hasBeenRetried) { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + if (isAttemptToFixRetry) { + span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + } const spanTags = span.context()._tags this.telemetry.ciVisEvent( @@ -403,18 +422,18 @@ class MochaPlugin extends CiPlugin { startTestSpan (testInfo) { const { + testName, testSuiteAbsolutePath, title, isNew, isEfdRetry, testStartLine, isParallel, + isAttemptToFix, isDisabled, isQuarantined } = testInfo - const testName = removeEfdStringFromTestName(testInfo.testName) - const extraTags = {} const testParametersString = getTestParametersString(this._testTitleToParams, title) if (testParametersString) { @@ -429,6 +448,10 @@ class MochaPlugin extends CiPlugin { extraTags[MOCHA_IS_PARALLEL] = 'true' } + if (isAttemptToFix) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + } + if (isDisabled) { extraTags[TEST_MANAGEMENT_IS_DISABLED] = 'true' } diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 9c75c690bc4..bc073a4f9d5 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -20,7 +20,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_ENABLED, TEST_BROWSER_NAME, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT } = require('../../dd-trace/src/constants') @@ -162,7 +165,11 @@ class PlaywrightPlugin extends CiPlugin { isNew, isEfdRetry, isRetry, - isQuarantined + isAttemptToFix, + isQuarantined, + isAttemptToFixRetry, + hasFailedAllRetries, + hasPassedAttemptToFixRetries }) => { const store = storage('legacy').getStore() const span = store && store.span @@ -186,6 +193,19 @@ class PlaywrightPlugin extends CiPlugin { if (isRetry) { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + if (isAttemptToFix) { + span.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + } + if (isAttemptToFixRetry) { + span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + } + if (hasPassedAttemptToFixRetries) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } if (isQuarantined) { span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') } diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index 06a5257d905..193ec152dc0 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -23,7 +23,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -53,18 +56,30 @@ class VitestPlugin extends CiPlugin { onDone(!testsForThisTestSuite.includes(testName)) }) + this.addSub('ci:vitest:test:is-attempt-to-fix', ({ + testManagementTests, + testSuiteAbsolutePath, + testName, + onDone + }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const { isAttemptToFix } = this.getTestProperties(testManagementTests, testSuite, testName) + + onDone(isAttemptToFix ?? false) + }) + this.addSub('ci:vitest:test:is-disabled', ({ testManagementTests, testSuiteAbsolutePath, testName, onDone }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const { isDisabled } = this.getTestProperties(testManagementTests, testSuite, testName) - onDone(isDisabled ?? false) + onDone(isDisabled) }) this.addSub('ci:vitest:test:is-quarantined', ({ testManagementTests, testSuiteAbsolutePath, testName, onDone }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const { isQuarantined } = this.getTestProperties(testManagementTests, testSuite, testName) - onDone(isQuarantined ?? false) + onDone(isQuarantined) }) this.addSub('ci:vitest:is-early-flake-detection-faulty', ({ @@ -85,9 +100,12 @@ class VitestPlugin extends CiPlugin { testSuiteAbsolutePath, isRetry, isNew, + isAttemptToFix, isQuarantined, + isDisabled, mightHitProbe, - isRetryReasonEfd + isRetryReasonEfd, + isRetryReasonAttemptToFix }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const store = storage('legacy').getStore() @@ -104,9 +122,18 @@ class VitestPlugin extends CiPlugin { if (isRetryReasonEfd) { extraTags[TEST_RETRY_REASON] = 'efd' } + if (isAttemptToFix) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + } + if (isRetryReasonAttemptToFix) { + extraTags[TEST_RETRY_REASON] = 'attempt_to_fix' + } if (isQuarantined) { extraTags[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' } + if (isDisabled) { + extraTags[TEST_MANAGEMENT_IS_DISABLED] = 'true' + } const span = this.startTestSpan( testName, @@ -124,7 +151,7 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:finish-time', ({ status, task }) => { + this.addSub('ci:vitest:test:finish-time', ({ status, task, attemptToFixPassed }) => { const store = storage('legacy').getStore() const span = store?.span @@ -132,6 +159,11 @@ class VitestPlugin extends CiPlugin { // this is because the test might fail at a `afterEach` hook if (span) { span.setTag(TEST_STATUS, status) + + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + this.taskToFinishTime.set(task, span._getTime()) } }) @@ -150,7 +182,7 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:error', ({ duration, error, shouldSetProbe, promises }) => { + this.addSub('ci:vitest:test:error', ({ duration, error, shouldSetProbe, promises, hasFailedAllRetries }) => { const store = storage('legacy').getStore() const span = store?.span @@ -172,6 +204,9 @@ class VitestPlugin extends CiPlugin { if (error) { span.setTag('error', error) } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } if (duration) { span.finish(span._startTime + duration - MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION) // milliseconds } else { @@ -327,10 +362,10 @@ class VitestPlugin extends CiPlugin { } getTestProperties (testManagementTests, testSuite, testName) { - const { disabled: isDisabled, quarantined: isQuarantined } = + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = testManagementTests?.vitest?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } } diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index a87078ed7e7..dbbcd2f0f0b 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -215,7 +215,8 @@ class CiVisibilityExporter extends AgentInfoExporter { isFlakyTestRetriesEnabled, isDiEnabled, isKnownTestsEnabled, - isTestManagementEnabled + isTestManagementEnabled, + testManagementAttemptToFixRetries } = remoteConfiguration return { isCodeCoverageEnabled, @@ -229,7 +230,9 @@ class CiVisibilityExporter extends AgentInfoExporter { flakyTestRetriesCount: this._config.flakyTestRetriesCount, isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled, isKnownTestsEnabled, - isTestManagementEnabled: isTestManagementEnabled && this._config.isTestManagementEnabled + isTestManagementEnabled: isTestManagementEnabled && this._config.isTestManagementEnabled, + testManagementAttemptToFixRetries: + testManagementAttemptToFixRetries ?? this._config.testManagementAttemptToFixRetries } } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index 39d9fd1e11b..ebd00ea7574 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -113,7 +113,9 @@ function getLibraryConfiguration ({ isFlakyTestRetriesEnabled, isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled, isKnownTestsEnabled, - isTestManagementEnabled: (testManagementConfig?.enabled ?? false) + isTestManagementEnabled: (testManagementConfig?.enabled ?? false), + testManagementAttemptToFixRetries: + testManagementConfig?.attempt_to_fix_retries } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) diff --git a/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js b/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js index 5aca25e9e19..f5e897f0782 100644 --- a/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js +++ b/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js @@ -6,7 +6,8 @@ function getTestManagementTests ({ isEvpProxy, evpProxyPrefix, isGzipCompatible, - repositoryUrl + repositoryUrl, + commitMessage }, done) { const options = { path: '/api/v2/test/libraries/test-management/tests', @@ -39,7 +40,8 @@ function getTestManagementTests ({ id: id().toString(10), type: 'ci_app_libraries_tests_request', attributes: { - repository_url: repositoryUrl + repository_url: repositoryUrl, + commit_message: commitMessage } } }) diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index e1940265706..63c716acb32 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -41,7 +41,14 @@ const { TELEMETRY_EVENT_CREATED, TELEMETRY_ITR_SKIPPED } = require('../ci-visibility/telemetry') -const { CI_PROVIDER_NAME, GIT_REPOSITORY_URL, GIT_COMMIT_SHA, GIT_BRANCH, CI_WORKSPACE_PATH } = require('./util/tags') +const { + CI_PROVIDER_NAME, + GIT_REPOSITORY_URL, + GIT_COMMIT_SHA, + GIT_BRANCH, + CI_WORKSPACE_PATH, + GIT_COMMIT_MESSAGE +} = require('./util/tags') const { OS_VERSION, OS_PLATFORM, OS_ARCHITECTURE, RUNTIME_NAME, RUNTIME_VERSION } = require('./util/env') const getDiClient = require('../ci-visibility/dynamic-instrumentation') @@ -251,7 +258,8 @@ module.exports = class CiPlugin extends Plugin { [RUNTIME_VERSION]: runtimeVersion, [GIT_BRANCH]: branch, [CI_PROVIDER_NAME]: ciProviderName, - [CI_WORKSPACE_PATH]: repositoryRoot + [CI_WORKSPACE_PATH]: repositoryRoot, + [GIT_COMMIT_MESSAGE]: commitMessage } = this.testEnvironmentMetadata this.repositoryRoot = repositoryRoot || process.cwd() @@ -269,7 +277,8 @@ module.exports = class CiPlugin extends Plugin { runtimeName, runtimeVersion, branch, - testLevel: 'suite' + testLevel: 'suite', + commitMessage } } diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 92d62424d48..c6e95d5a45b 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -58,6 +58,7 @@ const TEST_IS_RETRY = 'test.is_retry' const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled' const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason' const TEST_RETRY_REASON = 'test.retry_reason' +const TEST_HAS_FAILED_ALL_RETRIES = 'test.has_failed_all_retries' const CI_APP_ORIGIN = 'ciapp-test' @@ -120,9 +121,16 @@ const DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX = 'snapshot_id' const DI_DEBUG_ERROR_FILE_SUFFIX = 'file' const DI_DEBUG_ERROR_LINE_SUFFIX = 'line' +// Test Management tags +const TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX = 'test.test_management.is_attempt_to_fix' const TEST_MANAGEMENT_IS_DISABLED = 'test.test_management.is_test_disabled' const TEST_MANAGEMENT_IS_QUARANTINED = 'test.test_management.is_quarantined' const TEST_MANAGEMENT_ENABLED = 'test.test_management.enabled' +const TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED = 'test.test_management.attempt_to_fix_passed' + +// Test Management utils strings +const ATTEMPT_TO_FIX_STRING = "Retried by Datadog's Test Management" +const ATTEMPT_TEST_NAME_REGEX = new RegExp(ATTEMPT_TO_FIX_STRING + ' \\(#\\d+\\): ', 'g') module.exports = { TEST_CODE_OWNERS, @@ -155,6 +163,7 @@ module.exports = { TEST_EARLY_FLAKE_ENABLED, TEST_EARLY_FLAKE_ABORT_REASON, TEST_RETRY_REASON, + TEST_HAS_FAILED_ALL_RETRIES, getTestEnvironmentMetadata, getTestParametersString, finishAllTraceSpans, @@ -192,7 +201,9 @@ module.exports = { EFD_STRING, EFD_TEST_NAME_REGEX, removeEfdStringFromTestName, + removeAttemptToFixStringFromTestName, addEfdStringToTestName, + addAttemptToFixStringToTestName, getIsFaultyEarlyFlakeDetection, TEST_BROWSER_DRIVER, TEST_BROWSER_DRIVER_VERSION, @@ -212,9 +223,11 @@ module.exports = { DI_DEBUG_ERROR_LINE_SUFFIX, getFormattedError, DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_MANAGEMENT_IS_DISABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_ENABLED + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19 @@ -622,10 +635,18 @@ function addEfdStringToTestName (testName, numAttempt) { return `${EFD_STRING} (#${numAttempt}): ${testName}` } +function addAttemptToFixStringToTestName (testName, numAttempt) { + return `${ATTEMPT_TO_FIX_STRING} (#${numAttempt}): ${testName}` +} + function removeEfdStringFromTestName (testName) { return testName.replace(EFD_TEST_NAME_REGEX, '') } +function removeAttemptToFixStringFromTestName (testName) { + return testName.replace(ATTEMPT_TEST_NAME_REGEX, '') +} + function getIsFaultyEarlyFlakeDetection (projectSuites, testsBySuiteName, faultyThresholdPercentage) { let newSuites = 0 for (const suite of projectSuites) { From 0075b252b39c7a5f5960277053e9b187a21318d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= <60353145+Mariovido@users.noreply.github.com> Date: Tue, 25 Mar 2025 11:46:54 +0100 Subject: [PATCH 24/32] [test optimization] [SDTEST-1720] Change capabilities tagging to send version number (#5463) --- integration-tests/cucumber/cucumber.spec.js | 20 ++++---- integration-tests/cypress/cypress.spec.js | 28 +++++------ integration-tests/jest/jest.spec.js | 21 ++++---- integration-tests/mocha/mocha.spec.js | 50 ++++++++++++------- .../playwright/playwright.spec.js | 27 +++------- integration-tests/vitest/vitest.spec.js | 25 ++++------ .../datadog-instrumentations/src/vitest.js | 6 +-- .../src/cypress-plugin.js | 11 ++-- packages/datadog-plugin-vitest/src/index.js | 13 ++--- packages/dd-trace/src/plugins/ci_plugin.js | 27 ++-------- packages/dd-trace/src/plugins/util/test.js | 42 +++++++++++++++- 11 files changed, 137 insertions(+), 133 deletions(-) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 092dcbe3af7..4068aadbe60 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -50,6 +50,9 @@ const { DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, + DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE, + DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE, + DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_HAS_FAILED_ALL_RETRIES, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED @@ -2461,14 +2464,6 @@ versions.forEach(version => { runModes.forEach((runMode) => { it(`(${runMode}) adds capabilities to tests`, (done) => { - receiver.setSettings({ - flaky_test_retries_enabled: true, - itr_enabled: false, - early_flake_detection: { - enabled: true - }, - known_tests_enabled: false - }) const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand const receiverPromise = receiver @@ -2480,10 +2475,13 @@ versions.forEach(version => { if (runMode === 'parallel') { assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], undefined) } else { - assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], 'false') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], '1') } - assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], 'false') - assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], 'true') + assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1') + assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], '2') // capabilities logic does not overwrite test session name assert.equal(metadata.test[TEST_SESSION_NAME], 'my-test-session-name') }) diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 1ab83ca8064..3363ebcb2f9 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -42,13 +42,16 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_DISABLED, - DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, - DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES, TEST_NAME, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, - TEST_HAS_FAILED_ALL_RETRIES + TEST_HAS_FAILED_ALL_RETRIES, + DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, + DD_CAPABILITIES_EARLY_FLAKE_DETECTION, + DD_CAPABILITIES_AUTO_TEST_RETRIES, + DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE, + DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE, + DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2218,23 +2221,18 @@ moduleTypes.forEach(({ context('libraries capabilities', () => { it('adds capabilities to tests', (done) => { - receiver.setSettings({ - flaky_test_retries_enabled: false, - itr_enabled: false, - early_flake_detection: { - enabled: true - }, - known_tests_enabled: true - }) const receiverPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) assert.isNotEmpty(metadataDicts) metadataDicts.forEach(metadata => { - assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], 'false') - assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], 'true') - assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], 'false') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], '1') + assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1') + assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], '2') // capabilities logic does not overwrite test session name assert.equal(metadata.test[TEST_SESSION_NAME], 'my-test-session-name') }) diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index f2352437d2a..30068b4cdd3 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -47,6 +47,9 @@ const { DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, + DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE, + DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE, + DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_HAS_FAILED_ALL_RETRIES, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED @@ -3468,24 +3471,18 @@ describe('jest CommonJS', () => { context('libraries capabilities', () => { it('adds capabilities to tests', (done) => { - receiver.setSettings({ - flaky_test_retries_enabled: true, - itr_enabled: false, - early_flake_detection: { - enabled: true - }, - known_tests_enabled: true - }) - const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) assert.isNotEmpty(metadataDicts) metadataDicts.forEach(metadata => { - assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], 'false') - assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], 'true') - assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], 'true') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], '1') + assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1') + assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], '2') // capabilities logic does not overwrite test session name assert.equal(metadata.test[TEST_SESSION_NAME], 'my-test-session-name') }) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index c96d99953c4..0d324df4ce5 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -49,6 +49,9 @@ const { DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, + DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE, + DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE, + DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_HAS_FAILED_ALL_RETRIES, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED @@ -3041,44 +3044,55 @@ describe('mocha CommonJS', function () { }) context('libraries capabilities', () => { - it('adds capabilities to tests', (done) => { - receiver.setSettings({ - flaky_test_retries_enabled: true, - itr_enabled: true, - early_flake_detection: { - enabled: true - }, - known_tests_enabled: true - }) - - const eventsPromise = receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => { + const getTestAssertions = (isParallel) => + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => { const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) assert.isNotEmpty(metadataDicts) metadataDicts.forEach(metadata => { - assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], 'true') - assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], 'true') - assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], 'true') + if (isParallel) { + assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], undefined) + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], undefined) + } else { + assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], '2') + } + assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1') + assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], '1') // capabilities logic does not overwrite test session name assert.equal(metadata.test[TEST_SESSION_NAME], 'my-test-session-name') }) }) + const runTest = (done, isParallel, extraEnvVars = {}) => { + const testAssertionsPromise = getTestAssertions(isParallel) + childProcess = exec( runTestsWithCoverageCommand, { cwd, env: { ...getCiVisAgentlessConfig(receiver.port), - DD_TEST_SESSION_NAME: 'my-test-session-name' + DD_TEST_SESSION_NAME: 'my-test-session-name', + ...extraEnvVars }, stdio: 'inherit' } ) childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + testAssertionsPromise.then(() => done()).catch(done) + }) + } + + it('adds capabilities to tests', (done) => { + runTest(done, false) + }) + + it('adds capabilities to tests (parallel)', (done) => { + runTest(done, true, { + RUN_IN_PARALLEL: '1' }) }) }) diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 347ae5f980d..44c9baf8e27 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -34,6 +34,9 @@ const { DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, + DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE, + DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE, + DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_HAS_FAILED_ALL_RETRIES, TEST_NAME, @@ -1309,23 +1312,6 @@ versions.forEach((version) => { context('libraries capabilities', () => { it('adds capabilities to tests', (done) => { - receiver.setKnownTests( - { - playwright: { - 'passing-test.js': [ - 'should work with passing tests' - ] - } - } - ) - receiver.setSettings({ - flaky_test_retries_enabled: true, - itr_enabled: false, - early_flake_detection: { - enabled: true - }, - known_tests_enabled: true - }) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) @@ -1333,8 +1319,11 @@ versions.forEach((version) => { assert.isNotEmpty(metadataDicts) metadataDicts.forEach(metadata => { assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], undefined) - assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], 'true') - assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], 'true') + assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1') + assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], '2') // capabilities logic does not overwrite test session name assert.equal(metadata.test[TEST_SESSION_NAME], 'my-test-session-name') }) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index 3e26a1c9d03..1ef0c2e9b3f 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -35,12 +35,15 @@ const { TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, - TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, - TEST_HAS_FAILED_ALL_RETRIES, - TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED + DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE, + DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE, + DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -1782,15 +1785,6 @@ versions.forEach((version) => { context('libraries capabilities', () => { it('adds capabilities to tests', (done) => { - receiver.setSettings({ - flaky_test_retries_enabled: false, - itr_enabled: true, - early_flake_detection: { - enabled: true - }, - known_tests_enabled: true - }) - const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) @@ -1798,8 +1792,11 @@ versions.forEach((version) => { assert.isNotEmpty(metadataDicts) metadataDicts.forEach(metadata => { assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], undefined) - assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], 'true') - assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], 'false') + assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1') + assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], '1') + assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], '2') // capabilities logic does not overwrite test session name assert.equal(metadata.test[TEST_SESSION_NAME], 'my-test-session-name') }) diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index b3bf9809c4c..728b8a43001 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -721,15 +721,11 @@ addHook({ // From >=3.0.1, the first arguments changes from a string to an object containing the filepath const testSuiteAbsolutePath = testPaths[0]?.filepath || testPaths[0] - const { isEarlyFlakeDetectionEnabled, isFlakyTestRetriesEnabled } = getProvidedContext() - const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') testSuiteAsyncResource.runInAsyncScope(() => { testSuiteStartCh.publish({ testSuiteAbsolutePath, - frameworkVersion, - isFlakyTestRetriesEnabled, - isEarlyFlakeDetectionEnabled + frameworkVersion }) }) const startTestsResponse = await startTests.apply(this, arguments) diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 092db83a901..221807a6bd0 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -37,12 +37,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_DISABLED, - DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES, - DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, - TEST_HAS_FAILED_ALL_RETRIES + TEST_HAS_FAILED_ALL_RETRIES, + getLibraryCapabilitiesTags } = require('../../dd-trace/src/plugins/util/test') const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') @@ -482,11 +480,10 @@ class CypressPlugin { [TEST_SESSION_NAME]: testSessionName } } + const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id) metadataTags.test = { ...metadataTags.test, - [DD_CAPABILITIES_TEST_IMPACT_ANALYSIS]: this.isSuitesSkippingEnabled ? 'true' : 'false', - [DD_CAPABILITIES_EARLY_FLAKE_DETECTION]: this.isEarlyFlakeDetectionEnabled ? 'true' : 'false', - [DD_CAPABILITIES_AUTO_TEST_RETRIES]: this.isFlakyTestRetriesEnabled ? 'true' : 'false' + ...libraryCapabilitiesTags } this.tracer._tracer._exporter.addMetadataTags(metadataTags) diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index 193ec152dc0..3b3487b8164 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -22,11 +22,10 @@ const { TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_IS_DISABLED, - DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, - TEST_HAS_FAILED_ALL_RETRIES + TEST_HAS_FAILED_ALL_RETRIES, + getLibraryCapabilitiesTags } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -238,9 +237,7 @@ class VitestPlugin extends CiPlugin { this.addSub('ci:vitest:test-suite:start', ({ testSuiteAbsolutePath, - frameworkVersion, - isFlakyTestRetriesEnabled, - isEarlyFlakeDetectionEnabled + frameworkVersion }) => { this.command = process.env.DD_CIVISIBILITY_TEST_COMMAND this.frameworkVersion = frameworkVersion @@ -258,10 +255,10 @@ class VitestPlugin extends CiPlugin { } } if (this.tracer._exporter.addMetadataTags) { + const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id) metadataTags.test = { ...metadataTags.test, - [DD_CAPABILITIES_EARLY_FLAKE_DETECTION]: isEarlyFlakeDetectionEnabled ? 'true' : 'false', - [DD_CAPABILITIES_AUTO_TEST_RETRIES]: isFlakyTestRetriesEnabled ? 'true' : 'false' + ...libraryCapabilitiesTags } this.tracer._exporter.addMetadataTags(metadataTags) } diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 63c716acb32..5df8e3ca0af 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -28,9 +28,7 @@ const { DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_LINE_SUFFIX, - DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES, - DD_CAPABILITIES_TEST_IMPACT_ANALYSIS + getLibraryCapabilitiesTags } = require('./util/test') const Plugin = require('./plugin') const { COMPONENT } = require('../constants') @@ -52,19 +50,6 @@ const { const { OS_VERSION, OS_PLATFORM, OS_ARCHITECTURE, RUNTIME_NAME, RUNTIME_VERSION } = require('./util/env') const getDiClient = require('../ci-visibility/dynamic-instrumentation') -const UNSUPPORTED_TIA_FRAMEWORKS = ['playwright', 'vitest'] -const UNSUPPORTED_TIA_FRAMEWORKS_PARALLEL_MODE = ['cucumber', 'mocha'] - -function isTiaSupported (testFramework, isParallel) { - if (UNSUPPORTED_TIA_FRAMEWORKS.includes(testFramework)) { - return false - } - if (isParallel && UNSUPPORTED_TIA_FRAMEWORKS_PARALLEL_MODE.includes(testFramework)) { - return false - } - return true -} - module.exports = class CiPlugin extends Plugin { constructor (...args) { super(...args) @@ -82,17 +67,13 @@ module.exports = class CiPlugin extends Plugin { } else { this.libraryConfig = libraryConfig } - const { isItrEnabled, isEarlyFlakeDetectionEnabled, isFlakyTestRetriesEnabled } = this.libraryConfig || {} + + const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id, isParallel) const metadataTags = { test: { - [DD_CAPABILITIES_TEST_IMPACT_ANALYSIS]: isItrEnabled ? 'true' : 'false', - [DD_CAPABILITIES_EARLY_FLAKE_DETECTION]: isEarlyFlakeDetectionEnabled ? 'true' : 'false', - [DD_CAPABILITIES_AUTO_TEST_RETRIES]: isFlakyTestRetriesEnabled ? 'true' : 'false' + ...libraryCapabilitiesTags } } - if (!isTiaSupported(this.constructor.id, isParallel)) { - metadataTags.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS] = undefined - } this.tracer._exporter.addMetadataTags(metadataTags) onDone({ err, libraryConfig }) }) diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index c6e95d5a45b..9f0beb038e2 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -104,6 +104,12 @@ const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g') const DD_CAPABILITIES_TEST_IMPACT_ANALYSIS = '_dd.library_capabilities.test_impact_analysis' const DD_CAPABILITIES_EARLY_FLAKE_DETECTION = '_dd.library_capabilities.early_flake_detection' const DD_CAPABILITIES_AUTO_TEST_RETRIES = '_dd.library_capabilities.auto_test_retries' +const DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE = '_dd.library_capabilities.test_management.quarantine' +const DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE = '_dd.library_capabilities.test_management.disable' +const DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX = '_dd.library_capabilities.test_management.attempt_to_fix' +const UNSUPPORTED_TIA_FRAMEWORKS = ['playwright', 'vitest'] +const UNSUPPORTED_TIA_FRAMEWORKS_PARALLEL_MODE = ['cucumber', 'mocha'] +const UNSUPPORTED_ATTEMPT_TO_FIX_FRAMEWORKS_PARALLEL_MODE = ['mocha'] const TEST_LEVEL_EVENT_TYPES = [ 'test', @@ -213,6 +219,9 @@ module.exports = { DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, + DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE, + DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE, + DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX, TEST_LEVEL_EVENT_TYPES, getNumFromKnownTests, getFileAndLineNumberFromError, @@ -227,7 +236,8 @@ module.exports = { TEST_MANAGEMENT_IS_DISABLED, TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_ENABLED, - TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + getLibraryCapabilitiesTags } // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19 @@ -739,3 +749,33 @@ function getFormattedError (error, repositoryRoot) { return newError } + +function getLibraryCapabilitiesTags (testFramework, isParallel) { + function isTiaSupported (testFramework, isParallel) { + if (UNSUPPORTED_TIA_FRAMEWORKS.includes(testFramework)) { + return false + } + if (isParallel && UNSUPPORTED_TIA_FRAMEWORKS_PARALLEL_MODE.includes(testFramework)) { + return false + } + return true + } + + function isAttemptToFixSupported (testFramework, isParallel) { + if (isParallel && UNSUPPORTED_ATTEMPT_TO_FIX_FRAMEWORKS_PARALLEL_MODE.includes(testFramework)) { + return false + } + return true + } + + return { + [DD_CAPABILITIES_TEST_IMPACT_ANALYSIS]: isTiaSupported(testFramework, isParallel) ? '1' : undefined, + [DD_CAPABILITIES_EARLY_FLAKE_DETECTION]: '1', + [DD_CAPABILITIES_AUTO_TEST_RETRIES]: '1', + [DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE]: '1', + [DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE]: '1', + [DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX]: isAttemptToFixSupported(testFramework, isParallel) + ? '2' + : undefined + } +} From 23bdda69849fb2f27e3b7d2da546942b81582536 Mon Sep 17 00:00:00 2001 From: simon-id Date: Tue, 25 Mar 2025 13:39:41 +0100 Subject: [PATCH 25/32] Fix logging null when debugging and sending data without any error (#5480) Co-authored-by: FredericEspiau <7319147+FredericEspiau@users.noreply.github.com> --- packages/dd-trace/src/telemetry/send-data.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/dd-trace/src/telemetry/send-data.js b/packages/dd-trace/src/telemetry/send-data.js index 81406910c27..863e806d2b2 100644 --- a/packages/dd-trace/src/telemetry/send-data.js +++ b/packages/dd-trace/src/telemetry/send-data.js @@ -100,7 +100,11 @@ function sendData (config, application, host, reqType, payload = {}, cb = () => path: '/api/v2/apmtelemetry' } if (backendUrl) { - request(data, backendOptions, (error) => { log.error('Error sending telemetry data', error) }) + request(data, backendOptions, (error) => { + if (error) { + log.error('Error sending telemetry data', error) + } + }) } else { log.error('Invalid Telemetry URL') } From 5027915a1e746662a7fa13fa21c0d8602a3e0483 Mon Sep 17 00:00:00 2001 From: Sam Brenner <106700075+sabrenner@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:18:35 -0400 Subject: [PATCH 26/32] feat(llmobs): add vertexai plugin (#5413) * add llmobs plugin * wip tests * tests + ci workflow * move helper functions outside of class scope * add integration tag * Update packages/dd-trace/test/llmobs/plugins/google-cloud-vertexai/index.spec.js --- .github/workflows/llmobs.yml | 22 +- .../src/index.js | 194 +----------- .../src/tracing.js | 186 ++++++++++++ .../src/utils.js | 19 ++ .../dd-trace/src/llmobs/plugins/vertexai.js | 196 +++++++++++++ .../google-cloud-vertexai/index.spec.js | 275 ++++++++++++++++++ 6 files changed, 705 insertions(+), 187 deletions(-) create mode 100644 packages/datadog-plugin-google-cloud-vertexai/src/tracing.js create mode 100644 packages/datadog-plugin-google-cloud-vertexai/src/utils.js create mode 100644 packages/dd-trace/src/llmobs/plugins/vertexai.js create mode 100644 packages/dd-trace/test/llmobs/plugins/google-cloud-vertexai/index.spec.js diff --git a/.github/workflows/llmobs.yml b/.github/workflows/llmobs.yml index 0a689f5fc65..fc455cb4b3e 100644 --- a/.github/workflows/llmobs.yml +++ b/.github/workflows/llmobs.yml @@ -73,7 +73,7 @@ jobs: with: suffix: llmobs-${{ github.job }} - aws-sdk: + bedrock: runs-on: ubuntu-latest env: PLUGINS: aws-sdk @@ -92,3 +92,23 @@ jobs: uses: ./.github/actions/testagent/logs with: suffix: llmobs-${{ github.job }} + + vertex-ai: + runs-on: ubuntu-latest + env: + PLUGINS: google-cloud-vertexai + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/oldest-maintenance-lts + - uses: ./.github/actions/install + - run: yarn test:llmobs:plugins:ci + shell: bash + - uses: ./.github/actions/node/active-lts + - run: yarn test:llmobs:plugins:ci + shell: bash + - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 + - if: always() + uses: ./.github/actions/testagent/logs + with: + suffix: llmobs-${{ github.job }} diff --git a/packages/datadog-plugin-google-cloud-vertexai/src/index.js b/packages/datadog-plugin-google-cloud-vertexai/src/index.js index a8b2c1e3cfc..c2a0185432f 100644 --- a/packages/datadog-plugin-google-cloud-vertexai/src/index.js +++ b/packages/datadog-plugin-google-cloud-vertexai/src/index.js @@ -1,195 +1,17 @@ 'use strict' -const { MEASURED } = require('../../../ext/tags') -const { storage } = require('../../datadog-core') -const TracingPlugin = require('../../dd-trace/src/plugins/tracing') -const makeUtilities = require('../../dd-trace/src/plugins/util/llm') +const CompositePlugin = require('../../dd-trace/src/plugins/composite') +const GoogleVertexAITracingPlugin = require('./tracing') +const VertexAILLMObsPlugin = require('../../dd-trace/src/llmobs/plugins/vertexai') -class GoogleCloudVertexAIPlugin extends TracingPlugin { +class GoogleCloudVertexAIPlugin extends CompositePlugin { static get id () { return 'google-cloud-vertexai' } - static get prefix () { - return 'tracing:apm:vertexai:request' - } - - constructor () { - super(...arguments) - - Object.assign(this, makeUtilities('vertexai', this._tracerConfig)) - } - - bindStart (ctx) { - const { instance, request, resource, stream } = ctx - - const tags = this.tagRequest(request, instance, stream) - - const span = this.startSpan('vertexai.request', { - service: this.config.service, - resource, - kind: 'client', - meta: { - [MEASURED]: 1, - ...tags - } - }, false) - - const store = storage('legacy').getStore() || {} - ctx.currentStore = { ...store, span } - - return ctx.currentStore - } - - asyncEnd (ctx) { - const span = ctx.currentStore?.span - if (!span) return - - const { result } = ctx - - const response = result?.response - if (response) { - const tags = this.tagResponse(response) - span.addTags(tags) - } - - span.finish() - } - - tagRequest (request, instance, stream) { - const model = extractModel(instance) - const tags = { - 'vertexai.request.model': model - } - - const history = instance.historyInternal - let contents = typeof request === 'string' || Array.isArray(request) ? request : request.contents - if (history) { - contents = [...history, ...(Array.isArray(contents) ? contents : [contents])] - } - - const generationConfig = instance.generationConfig || {} - for (const key of Object.keys(generationConfig)) { - const transformedKey = key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase() - tags[`vertexai.request.generation_config.${transformedKey}`] = JSON.stringify(generationConfig[key]) - } - - if (stream) { - tags['vertexai.request.stream'] = true - } - - if (!this.isPromptCompletionSampled()) return tags - - const systemInstructions = extractSystemInstructions(instance) - - for (const [idx, systemInstruction] of systemInstructions.entries()) { - tags[`vertexai.request.system_instruction.${idx}.text`] = systemInstruction - } - - if (typeof contents === 'string') { - tags['vertexai.request.contents.0.text'] = contents - return tags - } - - for (const [contentIdx, content] of contents.entries()) { - this.tagRequestContent(tags, content, contentIdx) + static get plugins () { + return { + llmobs: VertexAILLMObsPlugin, + tracing: GoogleVertexAITracingPlugin } - - return tags } - - tagRequestPart (part, tags, partIdx, contentIdx) { - tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.text`] = this.normalize(part.text) - - const functionCall = part.functionCall - const functionResponse = part.functionResponse - - if (functionCall) { - tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.function_call.name`] = functionCall.name - tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.function_call.args`] = - this.normalize(JSON.stringify(functionCall.args)) - } - if (functionResponse) { - tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.function_response.name`] = - functionResponse.name - tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.function_response.response`] = - this.normalize(JSON.stringify(functionResponse.response)) - } - } - - tagRequestContent (tags, content, contentIdx) { - if (typeof content === 'string') { - tags[`vertexai.request.contents.${contentIdx}.text`] = this.normalize(content) - return - } - - if (content.text || content.functionCall || content.functionResponse) { - this.tagRequestPart(content, tags, 0, contentIdx) - return - } - - const { role, parts } = content - if (role) { - tags[`vertexai.request.contents.${contentIdx}.role`] = role - } - - for (const [partIdx, part] of parts.entries()) { - this.tagRequestPart(part, tags, partIdx, contentIdx) - } - } - - tagResponse (response) { - const tags = {} - - const candidates = response.candidates - for (const [candidateIdx, candidate] of candidates.entries()) { - const finishReason = candidate.finishReason - if (finishReason) { - tags[`vertexai.response.candidates.${candidateIdx}.finish_reason`] = finishReason - } - const candidateContent = candidate.content - const role = candidateContent.role - tags[`vertexai.response.candidates.${candidateIdx}.content.role`] = role - - if (!this.isPromptCompletionSampled()) continue - - const parts = candidateContent.parts - for (const [partIdx, part] of parts.entries()) { - const text = part.text - tags[`vertexai.response.candidates.${candidateIdx}.content.parts.${partIdx}.text`] = - this.normalize(String(text)) - - const functionCall = part.functionCall - if (!functionCall) continue - - tags[`vertexai.response.candidates.${candidateIdx}.content.parts.${partIdx}.function_call.name`] = - functionCall.name - tags[`vertexai.response.candidates.${candidateIdx}.content.parts.${partIdx}.function_call.args`] = - this.normalize(JSON.stringify(functionCall.args)) - } - } - - const tokenCounts = response.usageMetadata - if (tokenCounts) { - tags['vertexai.response.usage.prompt_tokens'] = tokenCounts.promptTokenCount - tags['vertexai.response.usage.completion_tokens'] = tokenCounts.candidatesTokenCount - tags['vertexai.response.usage.total_tokens'] = tokenCounts.totalTokenCount - } - - return tags - } -} - -function extractModel (instance) { - const model = instance.model || instance.resourcePath || instance.publisherModelEndpoint - return model?.split('/').pop() -} - -function extractSystemInstructions (instance) { - // systemInstruction is either a string or a Content object - // Content objects have parts (Part[]) and a role - const systemInstruction = instance.systemInstruction - if (!systemInstruction) return [] - if (typeof systemInstruction === 'string') return [systemInstruction] - - return systemInstruction.parts?.map(part => part.text) } module.exports = GoogleCloudVertexAIPlugin diff --git a/packages/datadog-plugin-google-cloud-vertexai/src/tracing.js b/packages/datadog-plugin-google-cloud-vertexai/src/tracing.js new file mode 100644 index 00000000000..41533fdd5a2 --- /dev/null +++ b/packages/datadog-plugin-google-cloud-vertexai/src/tracing.js @@ -0,0 +1,186 @@ +'use strict' + +const { MEASURED } = require('../../../ext/tags') +const { storage } = require('../../datadog-core') +const TracingPlugin = require('../../dd-trace/src/plugins/tracing') +const makeUtilities = require('../../dd-trace/src/plugins/util/llm') + +const { + extractModel, + extractSystemInstructions +} = require('./utils') + +class GoogleCloudVertexAITracingPlugin extends TracingPlugin { + static get id () { return 'google-cloud-vertexai' } + static get prefix () { + return 'tracing:apm:vertexai:request' + } + + constructor () { + super(...arguments) + + Object.assign(this, makeUtilities('vertexai', this._tracerConfig)) + } + + bindStart (ctx) { + const { instance, request, resource, stream } = ctx + + const tags = this.tagRequest(request, instance, stream) + + const span = this.startSpan('vertexai.request', { + service: this.config.service, + resource, + kind: 'client', + meta: { + [MEASURED]: 1, + ...tags + } + }, false) + + const store = storage('legacy').getStore() || {} + ctx.currentStore = { ...store, span } + + return ctx.currentStore + } + + asyncEnd (ctx) { + const span = ctx.currentStore?.span + if (!span) return + + const { result } = ctx + + const response = result?.response + if (response) { + const tags = this.tagResponse(response) + span.addTags(tags) + } + + span.finish() + } + + tagRequest (request, instance, stream) { + const model = extractModel(instance) + const tags = { + 'vertexai.request.model': model + } + + const history = instance.historyInternal + + let contents = typeof request === 'string' || Array.isArray(request) ? request : request.contents + if (history) { + contents = [...history, ...(Array.isArray(contents) ? contents : [contents])] + } + + const generationConfig = instance.generationConfig || {} + for (const key of Object.keys(generationConfig)) { + const transformedKey = key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase() + tags[`vertexai.request.generation_config.${transformedKey}`] = JSON.stringify(generationConfig[key]) + } + + if (stream) { + tags['vertexai.request.stream'] = true + } + + if (!this.isPromptCompletionSampled()) return tags + + const systemInstructions = extractSystemInstructions(instance) + + for (const [idx, systemInstruction] of systemInstructions.entries()) { + tags[`vertexai.request.system_instruction.${idx}.text`] = systemInstruction + } + + if (typeof contents === 'string') { + tags['vertexai.request.contents.0.text'] = contents + return tags + } + + for (const [contentIdx, content] of contents.entries()) { + this.tagRequestContent(tags, content, contentIdx) + } + + return tags + } + + tagRequestPart (part, tags, partIdx, contentIdx) { + tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.text`] = this.normalize(part.text) + + const functionCall = part.functionCall + const functionResponse = part.functionResponse + + if (functionCall) { + tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.function_call.name`] = functionCall.name + tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.function_call.args`] = + this.normalize(JSON.stringify(functionCall.args)) + } + if (functionResponse) { + tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.function_response.name`] = + functionResponse.name + tags[`vertexai.request.contents.${contentIdx}.parts.${partIdx}.function_response.response`] = + this.normalize(JSON.stringify(functionResponse.response)) + } + } + + tagRequestContent (tags, content, contentIdx) { + if (typeof content === 'string') { + tags[`vertexai.request.contents.${contentIdx}.text`] = this.normalize(content) + return + } + + if (content.text || content.functionCall || content.functionResponse) { + this.tagRequestPart(content, tags, 0, contentIdx) + return + } + + const { role, parts } = content + if (role) { + tags[`vertexai.request.contents.${contentIdx}.role`] = role + } + + for (const [partIdx, part] of parts.entries()) { + this.tagRequestPart(part, tags, partIdx, contentIdx) + } + } + + tagResponse (response) { + const tags = {} + + const candidates = response.candidates + for (const [candidateIdx, candidate] of candidates.entries()) { + const finishReason = candidate.finishReason + if (finishReason) { + tags[`vertexai.response.candidates.${candidateIdx}.finish_reason`] = finishReason + } + const candidateContent = candidate.content + const role = candidateContent.role + tags[`vertexai.response.candidates.${candidateIdx}.content.role`] = role + + if (!this.isPromptCompletionSampled()) continue + + const parts = candidateContent.parts + for (const [partIdx, part] of parts.entries()) { + const text = part.text + tags[`vertexai.response.candidates.${candidateIdx}.content.parts.${partIdx}.text`] = + this.normalize(String(text)) + + const functionCall = part.functionCall + if (!functionCall) continue + + tags[`vertexai.response.candidates.${candidateIdx}.content.parts.${partIdx}.function_call.name`] = + functionCall.name + tags[`vertexai.response.candidates.${candidateIdx}.content.parts.${partIdx}.function_call.args`] = + this.normalize(JSON.stringify(functionCall.args)) + } + } + + const tokenCounts = response.usageMetadata + if (tokenCounts) { + tags['vertexai.response.usage.prompt_tokens'] = tokenCounts.promptTokenCount + tags['vertexai.response.usage.completion_tokens'] = tokenCounts.candidatesTokenCount + tags['vertexai.response.usage.total_tokens'] = tokenCounts.totalTokenCount + } + + return tags + } +} + +module.exports = GoogleCloudVertexAITracingPlugin diff --git a/packages/datadog-plugin-google-cloud-vertexai/src/utils.js b/packages/datadog-plugin-google-cloud-vertexai/src/utils.js new file mode 100644 index 00000000000..81e6c7398f1 --- /dev/null +++ b/packages/datadog-plugin-google-cloud-vertexai/src/utils.js @@ -0,0 +1,19 @@ +function extractModel (instance) { + const model = instance.model || instance.resourcePath || instance.publisherModelEndpoint + return model?.split('/').pop() +} + +function extractSystemInstructions (instance) { + // systemInstruction is either a string or a Content object + // Content objects have parts (Part[]) and a role + const systemInstruction = instance.systemInstruction + if (!systemInstruction) return [] + if (typeof systemInstruction === 'string') return [systemInstruction] + + return systemInstruction.parts?.map(part => part.text) +} + +module.exports = { + extractModel, + extractSystemInstructions +} diff --git a/packages/dd-trace/src/llmobs/plugins/vertexai.js b/packages/dd-trace/src/llmobs/plugins/vertexai.js new file mode 100644 index 00000000000..78641b488c9 --- /dev/null +++ b/packages/dd-trace/src/llmobs/plugins/vertexai.js @@ -0,0 +1,196 @@ +'use strict' + +const LLMObsPlugin = require('./base') +const { + extractModel, + extractSystemInstructions +} = require('../../../../datadog-plugin-google-cloud-vertexai/src/utils') + +class VertexAILLMObsPlugin extends LLMObsPlugin { + static get id () { return 'vertexai' } // used for llmobs telemetry + static get prefix () { + return 'tracing:apm:vertexai:request' + } + + getLLMObsSpanRegisterOptions (ctx) { + const history = ctx.instance?.historyInternal || [] + ctx.history = history + + return { + kind: 'llm', + modelName: extractModel(ctx.instance), + modelProvider: 'google', + name: ctx.resource + } + } + + setLLMObsTags (ctx) { + const span = ctx.currentStore?.span + if (!span) return + + const { instance, result, request } = ctx + const history = ctx.history || [] + const systemInstructions = extractSystemInstructions(instance) + + const metadata = getMetadata(instance) + const inputMessages = extractInputMessages(request, history, systemInstructions) + const outputMessages = extractOutputMessages(result) + const metrics = extractMetrics(result) + + this._tagger.tagLLMIO(span, inputMessages, outputMessages) + this._tagger.tagMetadata(span, metadata) + this._tagger.tagMetrics(span, metrics) + } +} + +function getMetadata (instance) { + const metadata = {} + + const modelConfig = instance.generationConfig + if (!modelConfig) return metadata + + for (const [parameter, parameterKey] of [ + ['temperature', 'temperature'], + ['maxOutputTokens', 'max_output_tokens'], + ['candidateCount', 'candidate_count'], + ['topP', 'top_p'], + ['topK', 'top_k'] + ]) { + if (modelConfig[parameter]) { + metadata[parameterKey] = modelConfig[parameter] + } + } + + return metadata +} + +function extractInputMessages (request, history, systemInstructions) { + const contents = typeof request === 'string' || Array.isArray(request) ? request : request.contents + const messages = [] + + if (systemInstructions) { + for (const instruction of systemInstructions) { + messages.push({ content: instruction || '', role: 'system' }) + } + } + + for (const content of history) { + messages.push(...extractMessagesFromContent(content)) + } + + if (typeof contents === 'string') { + messages.push({ content: contents }) + return messages + } + + if (isPart(contents)) { + messages.push(extractMessageFromPart(contents)) + return messages + } + + if (!Array.isArray(contents)) { + messages.push({ + content: '[Non-array content object: ' + + `${(typeof contents.toString === 'function' ? contents.toString() : String(contents))}]` + }) + return messages + } + + for (const content of contents) { + if (typeof content === 'string') { + messages.push({ content }) + continue + } + + if (isPart(content)) { + messages.push(extractMessageFromPart(content)) + continue + } + + messages.push(...extractMessagesFromContent(content)) + } + + return messages +} + +function extractOutputMessages (result) { + if (!result) return [{ content: '' }] + const { response } = result + + if (!response) return [{ content: '' }] + + const outputMessages = [] + const candidates = response.candidates || [] + for (const candidate of candidates) { + const content = candidate.content || '' + outputMessages.push(...extractMessagesFromContent(content)) + } + + return outputMessages +} + +function extractMessagesFromContent (content) { + const messages = [] + + const role = content.role || '' + const parts = content.parts || [] + if (parts == null || parts.length === 0 || !Array.isArray(parts)) { + const message = { + content: + `[Non-text content object: ${(typeof content.toString === 'function' ? content.toString() : String(content))}]` + } + if (role) message.role = role + messages.push(message) + return messages + } + + for (const part of parts) { + const message = extractMessageFromPart(part, role) + messages.push(message) + } + + return messages +} + +function extractMessageFromPart (part, role) { + const text = part.text || '' + const functionCall = part.functionCall + const functionResponse = part.functionResponse + + const message = { content: text } + if (role) message.role = role + if (functionCall) { + message.toolCalls = [{ + name: functionCall.name, + arguments: functionCall.args + }] + } + if (functionResponse) { + message.content = `[tool result: ${functionResponse.response}]` + } + + return message +} + +function extractMetrics (result) { + if (!result) return {} + const { response } = result + + if (!response) return {} + + const tokenCounts = response.usageMetadata + const metrics = {} + if (tokenCounts) { + metrics.inputTokens = tokenCounts.promptTokenCount + metrics.outputTokens = tokenCounts.candidatesTokenCount + metrics.totalTokens = tokenCounts.totalTokenCount + } + + return metrics +} + +function isPart (part) { + return part.text || part.functionCall || part.functionResponse +} + +module.exports = VertexAILLMObsPlugin diff --git a/packages/dd-trace/test/llmobs/plugins/google-cloud-vertexai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/google-cloud-vertexai/index.spec.js new file mode 100644 index 00000000000..1cceb9e4073 --- /dev/null +++ b/packages/dd-trace/test/llmobs/plugins/google-cloud-vertexai/index.spec.js @@ -0,0 +1,275 @@ +'use strict' + +const LLMObsAgentProxySpanWriter = require('../../../../src/llmobs/writers/spans/agentProxy') +const agent = require('../../../plugins/agent') +const { + expectedLLMObsLLMSpanEvent, + deepEqualWithMockValues +} = require('../../util') +const chai = require('chai') + +const fs = require('node:fs') +const path = require('node:path') + +chai.Assertion.addMethod('deepEqualWithMockValues', deepEqualWithMockValues) + +/** + * @google-cloud/vertexai uses `fetch` to call against their API, which cannot + * be stubbed with `nock`. This function allows us to stub the `fetch` function + * to return a specific response for a given scenario. + * + * @param {string} scenario the scenario to load + * @param {number} statusCode the status code to return. defaults to 200 + */ +function useScenario ({ scenario, statusCode = 200, stream = false }) { + let originalFetch + + beforeEach(() => { + originalFetch = global.fetch + global.fetch = function () { + let body + + if (statusCode !== 200) { + body = '{}' + } else if (stream) { + body = fs.createReadStream(path.join( + __dirname, + '../../../../../datadog-plugin-google-cloud-vertexai/test/', + 'resources', + `${scenario}.txt`) + ) + } else { + const contents = require(`../../../../../datadog-plugin-google-cloud-vertexai/test/resources/${scenario}.json`) + body = JSON.stringify(contents) + } + + return new Response(body, { + status: statusCode, + headers: { + 'Content-Type': 'application/json' + } + }) + } + }) + + afterEach(() => { + global.fetch = originalFetch + }) +} + +describe('integrations', () => { + let authStub + let model + + function getInputMessages (content) { + const messages = [ + { role: 'user', content } + ] + + if (model.systemInstruction) { + // earlier versions of the SDK do not take a `systemInstruction` property + messages.unshift({ role: 'system', content: 'Please provide an answer' }) + } + + return messages + } + + describe('vertexai', () => { + before(async () => { + sinon.stub(LLMObsAgentProxySpanWriter.prototype, 'append') + + // reduce errors related to too many listeners + process.removeAllListeners('beforeExit') + + return agent.load('google-cloud-vertexai', {}, { + llmobs: { + mlApp: 'test' + } + }) + }) + + afterEach(() => { + LLMObsAgentProxySpanWriter.prototype.append.reset() + }) + + after(() => { + sinon.restore() + return agent.close({ ritmReset: false, wipe: true }) + }) + + withVersions('google-cloud-vertexai', '@google-cloud/vertexai', '>=1', version => { + before(() => { + const { VertexAI } = require(`../../../../../../versions/@google-cloud/vertexai@${version}`).get() + + // stub credentials checking + const { GoogleAuth } = require(`../../../../../../versions/@google-cloud/vertexai@${version}`) + .get('google-auth-library/build/src/auth/googleauth') + authStub = sinon.stub(GoogleAuth.prototype, 'getAccessToken').resolves({}) + + const client = new VertexAI({ + project: 'datadog-sandbox', + location: 'us-central1' + }) + + model = client.getGenerativeModel({ + model: 'gemini-1.5-flash-002', + systemInstruction: 'Please provide an answer', + generationConfig: { + maxOutputTokens: 50, + temperature: 1.0 + } + }) + }) + + after(() => { + authStub.restore() + }) + + describe('generateContent', () => { + useScenario({ scenario: 'generate-content-single-response' }) + + it('makes a successful call', async () => { + const checkTraces = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + modelName: 'gemini-1.5-flash-002', + modelProvider: 'google', + name: 'GenerativeModel.generateContent', + inputMessages: getInputMessages('Hello, how are you?'), + outputMessages: [ + { + role: 'model', + content: 'Hello! How can I assist you today?' + } + ], + metadata: { + temperature: 1, + max_output_tokens: 50 + }, + tokenMetrics: { input_tokens: 35, output_tokens: 2, total_tokens: 37 }, + tags: { ml_app: 'test', language: 'javascript', integration: 'vertexai' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await model.generateContent({ + contents: [{ role: 'user', parts: [{ text: 'Hello, how are you?' }] }] + }) + + await checkTraces + }) + }) + + describe('tool calls', () => { + useScenario({ scenario: 'generate-content-single-response-with-tools' }) + + it('makes a successful call', async () => { + const checkTraces = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + modelName: 'gemini-1.5-flash-002', + modelProvider: 'google', + name: 'GenerativeModel.generateContent', + inputMessages: getInputMessages('what is 2 + 2?'), + outputMessages: [ + { + role: 'model', + content: '', + tool_calls: [ + { + name: 'add', + arguments: { + a: 2, + b: 2 + } + } + ] + } + ], + metadata: { + temperature: 1, + max_output_tokens: 50 + }, + tokenMetrics: { input_tokens: 20, output_tokens: 3, total_tokens: 23 }, + tags: { ml_app: 'test', language: 'javascript', integration: 'vertexai' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await model.generateContent({ + contents: [{ role: 'user', parts: [{ text: 'what is 2 + 2?' }] }] + }) + + await checkTraces + }) + }) + + describe('chat model', () => { + describe('generateContent', () => { + useScenario({ scenario: 'generate-content-single-response' }) + + it('makes a successful call', async () => { + const checkTraces = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const inputMessages = [] + + if (model.systemInstruction) { + inputMessages.push({ role: 'system', content: 'Please provide an answer' }) + } + + inputMessages.push({ role: 'user', content: 'Foobar?' }) + inputMessages.push({ role: 'model', content: 'Foobar!' }) + inputMessages.push({ content: 'Hello, how are you?' }) + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + modelName: 'gemini-1.5-flash-002', + modelProvider: 'google', + name: 'ChatSession.sendMessage', + inputMessages, + outputMessages: [ + { + role: 'model', + content: 'Hello! How can I assist you today?' + } + ], + metadata: { + temperature: 1, + max_output_tokens: 50 + }, + tokenMetrics: { input_tokens: 35, output_tokens: 2, total_tokens: 37 }, + tags: { ml_app: 'test', language: 'javascript', integration: 'vertexai' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + const chat = model.startChat({ + history: [ + { role: 'user', parts: [{ text: 'Foobar?' }] }, + { role: 'model', parts: [{ text: 'Foobar!' }] } + ] + }) + + await chat.sendMessage([{ text: 'Hello, how are you?' }]) + + await checkTraces + }) + }) + }) + }) + }) +}) From bddd71db5ecf05cdbb741e6fb5b091bfdc83637f Mon Sep 17 00:00:00 2001 From: Ilyas Shabi Date: Tue, 25 Mar 2025 15:32:19 +0100 Subject: [PATCH 27/32] waf requests telemetry metrics (#5384) * waf requests telemetry metrics * fix undefined versions * add more tests * fix tags order * update waf metrics integration test * remove duplicate input_truncated on test * fix waf requests telemtry test * fix rate limiter * report waf error metric * fix updateWafRateLimitedMetric reporter test * fix functions order * change function names * add versions to report attack * fix reporting rate limiting metric --- packages/dd-trace/src/appsec/blocking.js | 2 + packages/dd-trace/src/appsec/graphql.js | 4 +- packages/dd-trace/src/appsec/reporter.js | 5 +- .../dd-trace/src/appsec/telemetry/common.js | 9 +- .../dd-trace/src/appsec/telemetry/index.js | 16 ++ packages/dd-trace/src/appsec/telemetry/waf.js | 24 +- .../dd-trace/test/appsec/blocking.spec.js | 15 +- packages/dd-trace/test/appsec/graphql.spec.js | 12 +- .../dd-trace/test/appsec/reporter.spec.js | 26 +- .../test/appsec/telemetry/rasp.spec.js | 7 +- .../test/appsec/telemetry/waf.spec.js | 98 +++++-- .../appsec/waf-metrics.integration.spec.js | 269 +++++++++++++----- 12 files changed, 374 insertions(+), 113 deletions(-) diff --git a/packages/dd-trace/src/appsec/blocking.js b/packages/dd-trace/src/appsec/blocking.js index 5b3dd290c00..ab9bafc3ae2 100644 --- a/packages/dd-trace/src/appsec/blocking.js +++ b/packages/dd-trace/src/appsec/blocking.js @@ -2,6 +2,7 @@ const log = require('../log') const blockedTemplates = require('./blocked_templates') +const { updateBlockFailureMetric } = require('./telemetry') const detectedSpecificEndpoints = {} @@ -128,6 +129,7 @@ function block (req, res, rootSpan, abortController, actionParameters = defaultB rootSpan?.setTag('_dd.appsec.block.failed', 1) log.error('[ASM] Blocking error', err) + updateBlockFailureMetric(req) return false } } diff --git a/packages/dd-trace/src/appsec/graphql.js b/packages/dd-trace/src/appsec/graphql.js index 2ff265e6282..4799bab6ce9 100644 --- a/packages/dd-trace/src/appsec/graphql.js +++ b/packages/dd-trace/src/appsec/graphql.js @@ -17,6 +17,7 @@ const { apolloChannel, apolloServerCoreChannel } = require('./channels') +const { updateBlockFailureMetric } = require('./telemetry') const graphqlRequestData = new WeakMap() @@ -106,8 +107,9 @@ function beforeWriteApolloGraphqlResponse ({ abortController, abortData }) { abortController?.abort() } catch (err) { rootSpan.setTag('_dd.appsec.block.failed', 1) - log.error('[ASM] Blocking error', err) + + updateBlockFailureMetric(req) } } diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index c5f3bdce56c..ba7ceaa7b0c 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -10,7 +10,8 @@ const { updateRaspRequestsMetricTags, incrementWafUpdatesMetric, incrementWafRequestsMetric, - getRequestMetrics + getRequestMetrics, + updateRateLimitedMetric } = require('./telemetry') const zlib = require('zlib') const { keepTrace } = require('../priority_sampler') @@ -149,6 +150,8 @@ function reportAttack (attackData) { if (limiter.isAllowed()) { keepTrace(rootSpan, ASM) + } else { + updateRateLimitedMetric(req) } // TODO: maybe add this to format.js later (to take decision as late as possible) diff --git a/packages/dd-trace/src/appsec/telemetry/common.js b/packages/dd-trace/src/appsec/telemetry/common.js index a8ed471bd10..d91cd9988b7 100644 --- a/packages/dd-trace/src/appsec/telemetry/common.js +++ b/packages/dd-trace/src/appsec/telemetry/common.js @@ -3,12 +3,15 @@ const DD_TELEMETRY_REQUEST_METRICS = Symbol('_dd.appsec.telemetry.request.metrics') const tags = { + BLOCK_FAILURE: 'block_failure', + EVENT_RULES_VERSION: 'event_rules_version', + INPUT_TRUNCATED: 'input_truncated', + RATE_LIMITED: 'rate_limited', REQUEST_BLOCKED: 'request_blocked', RULE_TRIGGERED: 'rule_triggered', + WAF_ERROR: 'waf_error', WAF_TIMEOUT: 'waf_timeout', - WAF_VERSION: 'waf_version', - EVENT_RULES_VERSION: 'event_rules_version', - INPUT_TRUNCATED: 'input_truncated' + WAF_VERSION: 'waf_version' } function getVersionsTags (wafVersion, rulesVersion) { diff --git a/packages/dd-trace/src/appsec/telemetry/index.js b/packages/dd-trace/src/appsec/telemetry/index.js index 0593efd97b9..18a3e7ca0ec 100644 --- a/packages/dd-trace/src/appsec/telemetry/index.js +++ b/packages/dd-trace/src/appsec/telemetry/index.js @@ -74,6 +74,20 @@ function updateWafRequestsMetricTags (metrics, req) { return trackWafMetrics(store, metrics) } +function updateRateLimitedMetric (req) { + if (!enabled) return + + const store = getStore(req) + trackWafMetrics(store, { rateLimited: true }) +} + +function updateBlockFailureMetric (req) { + if (!enabled) return + + const store = getStore(req) + trackWafMetrics(store, { blockFailed: true }) +} + function incrementWafInitMetric (wafVersion, rulesVersion, success) { if (!enabled) return @@ -119,6 +133,8 @@ module.exports = { disable, updateWafRequestsMetricTags, + updateRateLimitedMetric, + updateBlockFailureMetric, updateRaspRequestsMetricTags, incrementWafInitMetric, incrementWafUpdatesMetric, diff --git a/packages/dd-trace/src/appsec/telemetry/waf.js b/packages/dd-trace/src/appsec/telemetry/waf.js index bf995741ce2..5571c9cfa90 100644 --- a/packages/dd-trace/src/appsec/telemetry/waf.js +++ b/packages/dd-trace/src/appsec/telemetry/waf.js @@ -50,17 +50,28 @@ function trackWafMetrics (store, metrics) { const metricTags = getOrCreateMetricTags(store, versionsTags) - const { blockTriggered, ruleTriggered, wafTimeout } = metrics + if (metrics.blockFailed) { + metricTags[tags.BLOCK_FAILURE] = true + } - if (blockTriggered) { + if (metrics.blockTriggered) { metricTags[tags.REQUEST_BLOCKED] = true } - if (ruleTriggered) { + if (metrics.rateLimited) { + metricTags[tags.RATE_LIMITED] = true + } + + if (metrics.ruleTriggered) { metricTags[tags.RULE_TRIGGERED] = true } - if (wafTimeout) { + if (metrics.errorCode) { + metricTags[tags.WAF_ERROR] = true + appsecMetrics.count('waf.error', { ...versionsTags, waf_error: metrics.errorCode }).inc() + } + + if (metrics.wafTimeout) { metricTags[tags.WAF_TIMEOUT] = true } @@ -78,10 +89,13 @@ function getOrCreateMetricTags (store, versionsTags) { if (!metricTags) { metricTags = { + [tags.BLOCK_FAILURE]: false, + [tags.INPUT_TRUNCATED]: false, + [tags.RATE_LIMITED]: false, [tags.REQUEST_BLOCKED]: false, [tags.RULE_TRIGGERED]: false, + [tags.WAF_ERROR]: false, [tags.WAF_TIMEOUT]: false, - [tags.INPUT_TRUNCATED]: false, ...versionsTags } diff --git a/packages/dd-trace/test/appsec/blocking.spec.js b/packages/dd-trace/test/appsec/blocking.spec.js index 692ef58b6ef..a90d30dc9af 100644 --- a/packages/dd-trace/test/appsec/blocking.spec.js +++ b/packages/dd-trace/test/appsec/blocking.spec.js @@ -13,7 +13,7 @@ describe('blocking', () => { } } - let log + let log, telemetry let block, setTemplates let req, res, rootSpan @@ -22,9 +22,14 @@ describe('blocking', () => { warn: sinon.stub() } + telemetry = { + updateBlockFailureMetric: sinon.stub() + } + const blocking = proxyquire('../src/appsec/blocking', { '../log': log, - './blocked_templates': defaultBlockedTemplate + './blocked_templates': defaultBlockedTemplate, + './telemetry': telemetry }) block = blocking.block @@ -66,6 +71,7 @@ describe('blocking', () => { expect(rootSpan.setTag).to.have.been.calledOnceWithExactly('_dd.appsec.block.failed', 1) expect(res.setHeader).to.not.have.been.called expect(res.constructor.prototype.end).to.not.have.been.called + expect(telemetry.updateBlockFailureMetric).to.be.calledOnceWithExactly(req) }) it('should send blocking response with html type if present in the headers', () => { @@ -79,6 +85,7 @@ describe('blocking', () => { 'Content-Length': 12 }) expect(res.constructor.prototype.end).to.have.been.calledOnceWithExactly('htmlBodyéé') + expect(telemetry.updateBlockFailureMetric).to.not.have.been.called }) it('should send blocking response with json type if present in the headers in priority', () => { @@ -92,6 +99,7 @@ describe('blocking', () => { 'Content-Length': 8 }) expect(res.constructor.prototype.end).to.have.been.calledOnceWithExactly('jsonBody') + expect(telemetry.updateBlockFailureMetric).to.not.have.been.called }) it('should send blocking response with json type if neither html or json is present in the headers', () => { @@ -104,6 +112,7 @@ describe('blocking', () => { 'Content-Length': 8 }) expect(res.constructor.prototype.end).to.have.been.calledOnceWithExactly('jsonBody') + expect(telemetry.updateBlockFailureMetric).to.not.have.been.called }) it('should send blocking response and call abortController if passed in arguments', () => { @@ -118,6 +127,7 @@ describe('blocking', () => { }) expect(res.constructor.prototype.end).to.have.been.calledOnceWithExactly('jsonBody') expect(abortController.signal.aborted).to.be.true + expect(telemetry.updateBlockFailureMetric).to.not.have.been.called }) it('should remove all headers before sending blocking response', () => { @@ -135,6 +145,7 @@ describe('blocking', () => { 'Content-Length': 8 }) expect(res.constructor.prototype.end).to.have.been.calledOnceWithExactly('jsonBody') + expect(telemetry.updateBlockFailureMetric).to.not.have.been.called }) }) diff --git a/packages/dd-trace/test/appsec/graphql.spec.js b/packages/dd-trace/test/appsec/graphql.spec.js index ab030e3082b..9b6d6380e9e 100644 --- a/packages/dd-trace/test/appsec/graphql.spec.js +++ b/packages/dd-trace/test/appsec/graphql.spec.js @@ -12,8 +12,7 @@ const { } = require('../../src/appsec/channels') describe('GraphQL', () => { - let graphql - let blocking + let graphql, blocking, telemetry beforeEach(() => { const getBlockingData = sinon.stub() @@ -29,8 +28,13 @@ describe('GraphQL', () => { statusCode: 403 }) + telemetry = { + updateBlockFailureMetric: sinon.stub() + } + graphql = proxyquire('../../src/appsec/graphql', { - './blocking': blocking + './blocking': blocking, + './telemetry': telemetry }) }) @@ -234,6 +238,7 @@ describe('GraphQL', () => { expect(blocking.getBlockingData).to.have.been.calledOnceWithExactly(req, 'graphql', blockParameters) expect(rootSpan.setTag).to.have.been.calledOnceWithExactly('appsec.blocked', 'true') + expect(telemetry.updateBlockFailureMetric).to.not.have.been.called }) it('Should catch error when block fails', () => { @@ -263,6 +268,7 @@ describe('GraphQL', () => { expect(blocking.getBlockingData).to.have.been.calledOnceWithExactly(req, 'graphql', blockParameters) expect(rootSpan.setTag).to.have.been.calledOnceWithExactly('_dd.appsec.block.failed', 1) + expect(telemetry.updateBlockFailureMetric).to.be.calledOnceWithExactly(req) }) }) }) diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index 887f620b948..c5575c0baae 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -38,6 +38,7 @@ describe('reporter', () => { updateRaspRequestsMetricTags: sinon.stub(), incrementWafUpdatesMetric: sinon.stub(), incrementWafRequestsMetric: sinon.stub(), + updateRateLimitedMetric: sinon.stub(), getRequestMetrics: sinon.stub() } @@ -296,6 +297,7 @@ describe('reporter', () => { '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}' }) expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, ASM) + expect(telemetry.updateRateLimitedMetric).to.not.have.been.called }) it('should add tags to request span', () => { @@ -310,30 +312,35 @@ describe('reporter', () => { 'network.client.ip': '8.8.8.8' }) expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, ASM) + expect(telemetry.updateRateLimitedMetric).to.not.have.been.called }) it('should not add manual.keep when rate limit is reached', (done) => { const addTags = span.addTags - const params = {} - expect(Reporter.reportAttack('', params)).to.not.be.false - expect(Reporter.reportAttack('', params)).to.not.be.false - expect(Reporter.reportAttack('', params)).to.not.be.false + expect(Reporter.reportAttack('')).to.not.be.false + expect(Reporter.reportAttack('')).to.not.be.false + expect(Reporter.reportAttack('')).to.not.be.false expect(prioritySampler.setPriority).to.have.callCount(3) + expect(telemetry.updateRateLimitedMetric).to.not.have.been.called Reporter.setRateLimit(1) - expect(Reporter.reportAttack('', params)).to.not.be.false + expect(Reporter.reportAttack('')).to.not.be.false expect(addTags.getCall(3).firstArg).to.have.property('appsec.event').that.equals('true') expect(prioritySampler.setPriority).to.have.callCount(4) - expect(Reporter.reportAttack('', params)).to.not.be.false + expect(telemetry.updateRateLimitedMetric).to.not.have.been.called + + expect(Reporter.reportAttack('')).to.not.be.false expect(addTags.getCall(4).firstArg).to.have.property('appsec.event').that.equals('true') expect(prioritySampler.setPriority).to.have.callCount(4) + expect(telemetry.updateRateLimitedMetric).to.be.calledOnceWithExactly(req) setTimeout(() => { - expect(Reporter.reportAttack('', params)).to.not.be.false + expect(Reporter.reportAttack('')).to.not.be.false expect(prioritySampler.setPriority).to.have.callCount(5) + expect(telemetry.updateRateLimitedMetric).to.be.calledOnceWithExactly(req) done() }, 1020) }) @@ -341,7 +348,7 @@ describe('reporter', () => { it('should not overwrite origin tag', () => { span.context()._tags = { '_dd.origin': 'tracer' } - const result = Reporter.reportAttack('[]', {}) + const result = Reporter.reportAttack('[]') expect(result).to.not.be.false expect(web.root).to.have.been.calledOnceWith(req) @@ -351,6 +358,7 @@ describe('reporter', () => { 'network.client.ip': '8.8.8.8' }) expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, ASM) + expect(telemetry.updateRateLimitedMetric).to.not.have.been.called }) it('should merge attacks json', () => { @@ -367,6 +375,7 @@ describe('reporter', () => { 'network.client.ip': '8.8.8.8' }) expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, ASM) + expect(telemetry.updateRateLimitedMetric).to.not.have.been.called }) it('should call standalone sample', () => { @@ -384,6 +393,7 @@ describe('reporter', () => { }) expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, ASM) + expect(telemetry.updateRateLimitedMetric).to.not.have.been.called }) }) diff --git a/packages/dd-trace/test/appsec/telemetry/rasp.spec.js b/packages/dd-trace/test/appsec/telemetry/rasp.spec.js index c36ffd274c0..fcd20b78607 100644 --- a/packages/dd-trace/test/appsec/telemetry/rasp.spec.js +++ b/packages/dd-trace/test/appsec/telemetry/rasp.spec.js @@ -138,12 +138,15 @@ describe('Appsec Rasp Telemetry metrics', () => { appsecTelemetry.incrementWafRequestsMetric(req) expect(count).to.have.been.calledWithExactly('waf.requests', { + block_failure: false, + input_truncated: false, request_blocked: false, + rate_limited: false, rule_triggered: false, + waf_error: false, waf_timeout: false, waf_version: wafVersion, - event_rules_version: rulesVersion, - input_truncated: false + event_rules_version: rulesVersion }) }) }) diff --git a/packages/dd-trace/test/appsec/telemetry/waf.spec.js b/packages/dd-trace/test/appsec/telemetry/waf.spec.js index eff86ddabb6..d25f80ab112 100644 --- a/packages/dd-trace/test/appsec/telemetry/waf.spec.js +++ b/packages/dd-trace/test/appsec/telemetry/waf.spec.js @@ -53,12 +53,15 @@ describe('Appsec Waf Telemetry metrics', () => { const result = appsecTelemetry.updateWafRequestsMetricTags(metrics, req) expect(result).to.be.deep.eq({ - waf_version: wafVersion, + block_failure: false, event_rules_version: rulesVersion, + input_truncated: false, + rate_limited: false, request_blocked: false, rule_triggered: false, + waf_error: false, waf_timeout: false, - input_truncated: false + waf_version: wafVersion }) }) @@ -67,17 +70,22 @@ describe('Appsec Waf Telemetry metrics', () => { blockTriggered: true, ruleTriggered: true, wafTimeout: true, + rateLimited: true, + errorCode: -1, maxTruncatedString: 5000, ...metrics }, req) expect(result).to.be.deep.eq({ - waf_version: wafVersion, + block_failure: false, event_rules_version: rulesVersion, + input_truncated: true, + rate_limited: true, request_blocked: true, rule_triggered: true, + waf_error: true, waf_timeout: true, - input_truncated: true + waf_version: wafVersion }) }) @@ -86,18 +94,22 @@ describe('Appsec Waf Telemetry metrics', () => { const result2 = appsecTelemetry.updateWafRequestsMetricTags({ ruleTriggered: true, + rateLimited: true, ...metrics }, req) expect(result).to.be.eq(result2) expect(result).to.be.deep.eq({ - waf_version: wafVersion, + block_failure: false, event_rules_version: rulesVersion, + input_truncated: false, + rate_limited: true, request_blocked: false, rule_triggered: true, + waf_error: false, waf_timeout: false, - input_truncated: false + waf_version: wafVersion }) }) @@ -106,6 +118,7 @@ describe('Appsec Waf Telemetry metrics', () => { blockTriggered: true, ruleTriggered: true, wafTimeout: true, + rateLimited: true, maxTruncatedContainerSize: 300, ...metrics }, req) @@ -115,18 +128,22 @@ describe('Appsec Waf Telemetry metrics', () => { blockTriggered: false, ruleTriggered: false, wafTimeout: false, + rateLimited: false, ...metrics }, req2) expect(result).to.be.not.eq(result2) expect(result).to.be.deep.eq({ - waf_version: wafVersion, + block_failure: false, event_rules_version: rulesVersion, + input_truncated: true, + rate_limited: true, request_blocked: true, rule_triggered: true, + waf_error: false, waf_timeout: true, - input_truncated: true + waf_version: wafVersion }) }) @@ -175,8 +192,19 @@ describe('Appsec Waf Telemetry metrics', () => { }) it('should keep the maximum wafErrorCode', () => { - appsecTelemetry.updateWafRequestsMetricTags({ errorCode: -1 }, req) - appsecTelemetry.updateWafRequestsMetricTags({ errorCode: -3 }, req) + appsecTelemetry.updateWafRequestsMetricTags({ wafVersion, rulesVersion, errorCode: -1 }, req) + expect(count).to.have.been.calledWithExactly('waf.error', { + waf_version: wafVersion, + event_rules_version: rulesVersion, + waf_error: -1 + }) + + appsecTelemetry.updateWafRequestsMetricTags({ wafVersion, rulesVersion, errorCode: -3 }, req) + expect(count).to.have.been.calledWithExactly('waf.error', { + waf_version: wafVersion, + event_rules_version: rulesVersion, + waf_error: -3 + }) const { wafErrorCode } = appsecTelemetry.getRequestMetrics(req) expect(wafErrorCode).to.equal(-1) @@ -280,22 +308,30 @@ describe('Appsec Waf Telemetry metrics', () => { describe('incWafRequestsMetric', () => { it('should increment waf.requests metric', () => { appsecTelemetry.updateWafRequestsMetricTags({ - blockTriggered: false, - ruleTriggered: false, + blockTriggered: true, + blockFailed: true, + ruleTriggered: true, wafTimeout: true, + errorCode: -3, + rateLimited: true, + maxTruncatedString: 5000, wafVersion, rulesVersion }, req) appsecTelemetry.incrementWafRequestsMetric(req) - expect(count).to.have.been.calledOnceWithExactly('waf.requests', { - request_blocked: false, - rule_triggered: false, + expect(count).to.have.been.calledWithExactly('waf.input_truncated', { truncation_reason: 1 }) + expect(count).to.have.been.calledWithExactly('waf.requests', { + request_blocked: true, + block_failure: true, + rule_triggered: true, waf_timeout: true, + waf_error: true, + rate_limited: true, + input_truncated: true, waf_version: wafVersion, - event_rules_version: rulesVersion, - input_truncated: false + event_rules_version: rulesVersion }) }) @@ -306,6 +342,22 @@ describe('Appsec Waf Telemetry metrics', () => { }) }) + describe('updateRateLimitedMetric', () => { + it('should set rate_limited to true on the request tags', () => { + appsecTelemetry.updateRateLimitedMetric(req, metrics) + const result = appsecTelemetry.updateWafRequestsMetricTags({ wafVersion, rulesVersion }, req) + expect(result.rate_limited).to.be.true + }) + }) + + describe('updateBlockFailureMetric', () => { + it('should set block_failure to true on the request tags', () => { + appsecTelemetry.updateBlockFailureMetric(req) + const result = appsecTelemetry.updateWafRequestsMetricTags({ wafVersion, rulesVersion }, req) + expect(result.block_failure).to.be.true + }) + }) + describe('WAF Truncation metrics', () => { it('should report truncated string metrics', () => { const result = appsecTelemetry.updateWafRequestsMetricTags({ maxTruncatedString: 5000 }, req) @@ -389,6 +441,18 @@ describe('Appsec Waf Telemetry metrics', () => { expect(inc).to.not.have.been.called }) + it('should not set rate_limited if telemetry is disabled', () => { + appsecTelemetry.updateRateLimitedMetric(req, { wafVersion, rulesVersion }) + const result = appsecTelemetry.updateWafRequestsMetricTags({ wafVersion, rulesVersion }, req) + expect(result).to.be.undefined + }) + + it('should not set block_failure if telemetry is disabled', () => { + appsecTelemetry.updateBlockFailureMetric(req) + const result = appsecTelemetry.updateWafRequestsMetricTags({ wafVersion, rulesVersion }, req) + expect(result).to.be.undefined + }) + describe('updateWafRequestMetricTags', () => { it('should sum waf.duration and waf.durationExt request metrics', () => { appsecTelemetry.enable({ diff --git a/packages/dd-trace/test/appsec/waf-metrics.integration.spec.js b/packages/dd-trace/test/appsec/waf-metrics.integration.spec.js index 322ce7a2fb2..268d3c972b7 100644 --- a/packages/dd-trace/test/appsec/waf-metrics.integration.spec.js +++ b/packages/dd-trace/test/appsec/waf-metrics.integration.spec.js @@ -6,8 +6,8 @@ const path = require('path') const Axios = require('axios') const { assert } = require('chai') -describe('WAF truncation metrics', () => { - let axios, sandbox, cwd, appPort, appFile, agent, proc +describe('WAF Metrics', () => { + let axios, sandbox, cwd, appPort, appFile before(async function () { this.timeout(process.platform === 'win32' ? 90000 : 30000) @@ -32,96 +32,223 @@ describe('WAF truncation metrics', () => { await sandbox.remove() }) - beforeEach(async () => { - agent = await new FakeAgent().start() - proc = await spawnProc(appFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: agent.port, - APP_PORT: appPort, - DD_APPSEC_ENABLED: 'true', - DD_TELEMETRY_HEARTBEAT_INTERVAL: 1 + describe('WAF error metrics', () => { + let agent, proc + + beforeEach(async () => { + agent = await new FakeAgent().start() + proc = await spawnProc(appFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + APP_PORT: appPort, + DD_APPSEC_ENABLED: 'true', + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, + DD_APPSEC_WAF_TIMEOUT: 0.1 + } + }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('should report waf error metrics', async () => { + let appsecTelemetryMetricsReceived = false + + const body = { + name: 'hey' } + + await axios.post('/', body) + + const checkMessages = agent.assertMessageReceived(({ payload }) => { + assert.strictEqual(payload[0][0].metrics['_dd.appsec.enabled'], 1) + assert.strictEqual(payload[0][0].metrics['_dd.appsec.waf.error'], -127) + }) + + const checkTelemetryMetrics = agent.assertTelemetryReceived(({ payload }) => { + const namespace = payload.payload.namespace + + if (namespace === 'appsec') { + appsecTelemetryMetricsReceived = true + const series = payload.payload.series + const wafRequests = series.find(s => s.metric === 'waf.requests') + + assert.exists(wafRequests, 'Waf requests serie should exist') + assert.strictEqual(wafRequests.type, 'count') + assert.include(wafRequests.tags, 'waf_error:true') + assert.include(wafRequests.tags, 'rate_limited:false') + + const wafError = series.find(s => s.metric === 'waf.error') + assert.exists(wafError, 'Waf error serie should exist') + assert.strictEqual(wafError.type, 'count') + assert.include(wafError.tags, 'waf_error:-127') + } + }, 30_000, 'generate-metrics', 2) + + return Promise.all([checkMessages, checkTelemetryMetrics]).then(() => { + assert.equal(appsecTelemetryMetricsReceived, true) + + return true + }) }) }) - afterEach(async () => { - proc.kill() - await agent.stop() + describe('WAF timeout metrics', () => { + let agent, proc + + beforeEach(async () => { + agent = await new FakeAgent().start() + proc = await spawnProc(appFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + APP_PORT: appPort, + DD_APPSEC_ENABLED: 'true', + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, + DD_APPSEC_WAF_TIMEOUT: 1 + } + }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('should report waf timeout metrics', async () => { + let appsecTelemetryMetricsReceived = false + + const complexPayload = createComplexPayload() + await axios.post('/', { complexPayload }) + + const checkMessages = agent.assertMessageReceived(({ payload }) => { + assert.strictEqual(payload[0][0].metrics['_dd.appsec.enabled'], 1) + assert.isTrue(payload[0][0].metrics['_dd.appsec.waf.timeouts'] > 0) + }) + + const checkTelemetryMetrics = agent.assertTelemetryReceived(({ payload }) => { + const namespace = payload.payload.namespace + + if (namespace === 'appsec') { + appsecTelemetryMetricsReceived = true + const series = payload.payload.series + const wafRequests = series.find(s => s.metric === 'waf.requests') + + assert.exists(wafRequests, 'Waf requests serie should exist') + assert.strictEqual(wafRequests.type, 'count') + assert.include(wafRequests.tags, 'waf_timeout:true') + } + }, 30_000, 'generate-metrics', 2) + + return Promise.all([checkMessages, checkTelemetryMetrics]).then(() => { + assert.equal(appsecTelemetryMetricsReceived, true) + + return true + }) + }) }) - it('should report tuncation metrics', async () => { - let appsecTelemetryMetricsReceived = false - let appsecTelemetryDistributionsReceived = false - - const longValue = 'testattack'.repeat(500) - const largeObject = {} - for (let i = 0; i < 300; ++i) { - largeObject[`key${i}`] = `value${i}` - } - const deepObject = createNestedObject(25, { value: 'a' }) - const complexPayload = { - deepObject, - longValue, - largeObject - } - - await axios.post('/', { complexPayload }) - - const checkMessages = agent.assertMessageReceived(({ payload }) => { - assert.strictEqual(payload[0][0].metrics['_dd.appsec.enabled'], 1) - assert.strictEqual(payload[0][0].metrics['_dd.appsec.truncated.container_depth'], 20) - assert.strictEqual(payload[0][0].metrics['_dd.appsec.truncated.container_size'], 300) - assert.strictEqual(payload[0][0].metrics['_dd.appsec.truncated.string_length'], 5000) + describe('WAF truncation metrics', () => { + let agent, proc + + beforeEach(async () => { + agent = await new FakeAgent().start() + proc = await spawnProc(appFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + APP_PORT: appPort, + DD_APPSEC_ENABLED: 'true', + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1 + } + }) }) - const checkTelemetryMetrics = agent.assertTelemetryReceived(({ payload }) => { - const namespace = payload.payload.namespace + afterEach(async () => { + proc.kill() + await agent.stop() + }) - if (namespace === 'appsec') { - appsecTelemetryMetricsReceived = true - const series = payload.payload.series - const inputTruncated = series.find(s => s.metric === 'waf.input_truncated') + it('should report truncation metrics', async () => { + let appsecTelemetryMetricsReceived = false + let appsecTelemetryDistributionsReceived = false - assert.exists(inputTruncated, 'input truncated serie should exist') - assert.strictEqual(inputTruncated.type, 'count') - assert.include(inputTruncated.tags, 'truncation_reason:7') + const complexPayload = createComplexPayload() + await axios.post('/', { complexPayload }) - const wafRequests = series.find(s => s.metric === 'waf.requests') - assert.exists(wafRequests, 'waf requests serie should exist') - assert.include(wafRequests.tags, 'input_truncated:true') - } - }, 30_000, 'generate-metrics', 2) + const checkMessages = agent.assertMessageReceived(({ payload }) => { + assert.strictEqual(payload[0][0].metrics['_dd.appsec.enabled'], 1) + assert.strictEqual(payload[0][0].metrics['_dd.appsec.truncated.container_depth'], 20) + assert.strictEqual(payload[0][0].metrics['_dd.appsec.truncated.container_size'], 300) + assert.strictEqual(payload[0][0].metrics['_dd.appsec.truncated.string_length'], 5000) + }) - const checkTelemetryDistributions = agent.assertTelemetryReceived(({ payload }) => { - const namespace = payload.payload.namespace + const checkTelemetryMetrics = agent.assertTelemetryReceived(({ payload }) => { + const namespace = payload.payload.namespace - if (namespace === 'appsec') { - appsecTelemetryDistributionsReceived = true - const series = payload.payload.series - const wafDuration = series.find(s => s.metric === 'waf.duration') - const wafDurationExt = series.find(s => s.metric === 'waf.duration_ext') - const wafTuncated = series.filter(s => s.metric === 'waf.truncated_value_size') + if (namespace === 'appsec') { + appsecTelemetryMetricsReceived = true + const series = payload.payload.series + const inputTruncated = series.find(s => s.metric === 'waf.input_truncated') - assert.exists(wafDuration, 'waf duration serie should exist') - assert.exists(wafDurationExt, 'waf duration ext serie should exist') + assert.exists(inputTruncated, 'input truncated serie should exist') + assert.strictEqual(inputTruncated.type, 'count') + assert.include(inputTruncated.tags, 'truncation_reason:7') - assert.equal(wafTuncated.length, 3) - assert.include(wafTuncated[0].tags, 'truncation_reason:1') - assert.include(wafTuncated[1].tags, 'truncation_reason:2') - assert.include(wafTuncated[2].tags, 'truncation_reason:4') - } - }, 30_000, 'distributions', 1) + const wafRequests = series.find(s => s.metric === 'waf.requests') + assert.exists(wafRequests, 'waf requests serie should exist') + assert.include(wafRequests.tags, 'input_truncated:true') + } + }, 30_000, 'generate-metrics', 2) + + const checkTelemetryDistributions = agent.assertTelemetryReceived(({ payload }) => { + const namespace = payload.payload.namespace + + if (namespace === 'appsec') { + appsecTelemetryDistributionsReceived = true + const series = payload.payload.series + const wafDuration = series.find(s => s.metric === 'waf.duration') + const wafDurationExt = series.find(s => s.metric === 'waf.duration_ext') + const wafTuncated = series.filter(s => s.metric === 'waf.truncated_value_size') + + assert.exists(wafDuration, 'waf duration serie should exist') + assert.exists(wafDurationExt, 'waf duration ext serie should exist') + + assert.equal(wafTuncated.length, 3) + assert.include(wafTuncated[0].tags, 'truncation_reason:1') + assert.include(wafTuncated[1].tags, 'truncation_reason:2') + assert.include(wafTuncated[2].tags, 'truncation_reason:4') + } + }, 30_000, 'distributions', 1) - return Promise.all([checkMessages, checkTelemetryMetrics, checkTelemetryDistributions]).then(() => { - assert.equal(appsecTelemetryMetricsReceived, true) - assert.equal(appsecTelemetryDistributionsReceived, true) + return Promise.all([checkMessages, checkTelemetryMetrics, checkTelemetryDistributions]).then(() => { + assert.equal(appsecTelemetryMetricsReceived, true) + assert.equal(appsecTelemetryDistributionsReceived, true) - return true + return true + }) }) }) }) +const createComplexPayload = () => { + const longValue = 'testattack'.repeat(500) + const largeObject = {} + for (let i = 0; i < 300; ++i) { + largeObject[`key${i}`] = `value${i}` + } + const deepObject = createNestedObject(25, { value: 'a' }) + + return { + deepObject, + longValue, + largeObject + } +} + const createNestedObject = (n, obj) => { if (n > 0) { return { a: createNestedObject(n - 1, obj) } From a385cacfa159c1e89587f53e0f03431a2ea2b68e Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Tue, 25 Mar 2025 12:04:25 -0400 Subject: [PATCH 28/32] fix memory leak in runtime.node.heap.* metrics (#5476) * fix memory leak in runtime.node.heap metrics * switch to promise-based timers for test * add benchmark --- benchmark/sirun/runtime-metrics/README.md | 5 + benchmark/sirun/runtime-metrics/index.js | 5 + benchmark/sirun/runtime-metrics/meta.json | 22 +++ .../src/runtime_metrics/runtime_metrics.js | 13 +- .../dd-trace/test/runtime_metrics.spec.js | 154 ++++++++++-------- 5 files changed, 125 insertions(+), 74 deletions(-) create mode 100644 benchmark/sirun/runtime-metrics/README.md create mode 100644 benchmark/sirun/runtime-metrics/index.js create mode 100644 benchmark/sirun/runtime-metrics/meta.json diff --git a/benchmark/sirun/runtime-metrics/README.md b/benchmark/sirun/runtime-metrics/README.md new file mode 100644 index 00000000000..8f4c763436e --- /dev/null +++ b/benchmark/sirun/runtime-metrics/README.md @@ -0,0 +1,5 @@ +This benchmark runs the with runtime metrics and an accelerated flush. While +this can catch code regressions, it's mostly meant to catch things like memory +leaks where metrics would start piling up. This can be hard to catch in tests, +but it would cause the app to become slower and slower over time which would +be visible in the benchmark. diff --git a/benchmark/sirun/runtime-metrics/index.js b/benchmark/sirun/runtime-metrics/index.js new file mode 100644 index 00000000000..57024f27d47 --- /dev/null +++ b/benchmark/sirun/runtime-metrics/index.js @@ -0,0 +1,5 @@ +'use strict' + +require('../../..').init() + +setTimeout(() => {}, 1000) diff --git a/benchmark/sirun/runtime-metrics/meta.json b/benchmark/sirun/runtime-metrics/meta.json new file mode 100644 index 00000000000..45e37c08548 --- /dev/null +++ b/benchmark/sirun/runtime-metrics/meta.json @@ -0,0 +1,22 @@ +{ + "name": "runtime-metrics", + "run": "node index.js", + "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"", + "cachegrind": false, + "iterations": 5, + "instructions": true, + "variants": { + "control": { + "env": { + "DD_RUNTIME_METRICS_ENABLED": "false" + } + }, + "with-runtime-metrics": { + "baseline": "control", + "env": { + "DD_RUNTIME_METRICS_ENABLED": "true", + "DD_RUNTIME_METRICS_FLUSH_INTERVAL": "20" + } + } + } +} diff --git a/packages/dd-trace/src/runtime_metrics/runtime_metrics.js b/packages/dd-trace/src/runtime_metrics/runtime_metrics.js index 57177bef16a..8eb053646bc 100644 --- a/packages/dd-trace/src/runtime_metrics/runtime_metrics.js +++ b/packages/dd-trace/src/runtime_metrics/runtime_metrics.js @@ -10,7 +10,8 @@ const Histogram = require('../histogram') const { performance, PerformanceObserver } = require('perf_hooks') const { NODE_MAJOR, NODE_MINOR } = require('../../../../version') -const INTERVAL = 10 * 1000 +const { DD_RUNTIME_METRICS_FLUSH_INTERVAL = '10000' } = process.env +const INTERVAL = parseInt(DD_RUNTIME_METRICS_FLUSH_INTERVAL, 10) // Node >=16 has PerformanceObserver with `gc` type, but <16.7 had a critical bug. // See: https://github.com/nodejs/node/issues/39548 @@ -270,12 +271,12 @@ function captureNativeMetrics () { }) for (let i = 0, l = spaces.length; i < l; i++) { - const tags = [`heap_space:${spaces[i].space_name}`] + const tag = `heap_space:${spaces[i].space_name}` - client.gauge('runtime.node.heap.size.by.space', spaces[i].space_size, tags) - client.gauge('runtime.node.heap.used_size.by.space', spaces[i].space_used_size, tags) - client.gauge('runtime.node.heap.available_size.by.space', spaces[i].space_available_size, tags) - client.gauge('runtime.node.heap.physical_size.by.space', spaces[i].physical_space_size, tags) + client.gauge('runtime.node.heap.size.by.space', spaces[i].space_size, tag) + client.gauge('runtime.node.heap.used_size.by.space', spaces[i].space_used_size, tag) + client.gauge('runtime.node.heap.available_size.by.space', spaces[i].space_available_size, tag) + client.gauge('runtime.node.heap.physical_size.by.space', spaces[i].physical_space_size, tag) } } diff --git a/packages/dd-trace/test/runtime_metrics.spec.js b/packages/dd-trace/test/runtime_metrics.spec.js index 612c1e7f423..1ead31caf17 100644 --- a/packages/dd-trace/test/runtime_metrics.spec.js +++ b/packages/dd-trace/test/runtime_metrics.spec.js @@ -146,7 +146,7 @@ suiteDescribe('runtimeMetrics', () => { } } - setImmediate = globalThis.setImmediate + setImmediate = require('timers/promises').setImmediate clock = sinon.useFakeTimers() runtimeMetrics.start(config) @@ -188,79 +188,97 @@ suiteDescribe('runtimeMetrics', () => { }) }) - it('should start collecting runtimeMetrics every 10 seconds', (done) => { + it('should start collecting runtimeMetrics every 10 seconds', async () => { runtimeMetrics.stop() runtimeMetrics.start(config) global.gc() - setImmediate(() => setImmediate(() => { // Wait for GC observer to trigger. - clock.tick(10000) + // Wait for GC observer to trigger. + await setImmediate() + await setImmediate() + + clock.tick(10000) + + expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.user') + expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.system') + expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.total') + + expect(client.gauge).to.have.been.calledWith('runtime.node.mem.rss') + expect(client.gauge).to.have.been.calledWith('runtime.node.mem.heap_total') + expect(client.gauge).to.have.been.calledWith('runtime.node.mem.heap_used') + + expect(client.gauge).to.have.been.calledWith('runtime.node.process.uptime') + + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size_executable') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_physical_size') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_available_size') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.heap_size_limit') + + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.malloced_memory') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.peak_malloced_memory') + + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.max', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.min', sinon.match.number) + expect(client.increment).to.have.been.calledWith('runtime.node.event_loop.delay.sum', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.avg', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.median', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.95percentile', sinon.match.number) + expect(client.increment).to.have.been.calledWith('runtime.node.event_loop.delay.count', sinon.match.number) + + expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.utilization') + + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.max', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.min', sinon.match.number) + expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.sum', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.avg', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.median', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.95percentile', sinon.match.number) + expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.count', sinon.match.number) + + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.max', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.min', sinon.match.number) + expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.by.type.sum', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.avg', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.median', sinon.match.number) + expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.95percentile', sinon.match.number) + expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.by.type.count', sinon.match.number) + expect(client.increment).to.have.been.calledWith( + 'runtime.node.gc.pause.by.type.count', sinon.match.any, sinon.match(val => { + return val && /^gc_type:[a-z_]+$/.test(val[0]) + }) + ) + + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.size.by.space') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.used_size.by.space') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.available_size.by.space') + expect(client.gauge).to.have.been.calledWith('runtime.node.heap.physical_size.by.space') + + expect(client.flush).to.have.been.called + }) + + it('should collect individual metrics only once every 10 seconds', async () => { + runtimeMetrics.stop() + runtimeMetrics.start(config) + + global.gc() + + // Wait for GC observer to trigger. + await setImmediate() + await setImmediate() + + clock.tick(60 * 60 * 1000) - try { - expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.user') - expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.system') - expect(client.gauge).to.have.been.calledWith('runtime.node.cpu.total') - - expect(client.gauge).to.have.been.calledWith('runtime.node.mem.rss') - expect(client.gauge).to.have.been.calledWith('runtime.node.mem.heap_total') - expect(client.gauge).to.have.been.calledWith('runtime.node.mem.heap_used') - - expect(client.gauge).to.have.been.calledWith('runtime.node.process.uptime') - - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size_executable') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_physical_size') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_available_size') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.total_heap_size') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.heap_size_limit') - - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.malloced_memory') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.peak_malloced_memory') - - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.max', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.min', sinon.match.number) - expect(client.increment).to.have.been.calledWith('runtime.node.event_loop.delay.sum', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.avg', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.median', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.delay.95percentile', sinon.match.number) - expect(client.increment).to.have.been.calledWith('runtime.node.event_loop.delay.count', sinon.match.number) - - expect(client.gauge).to.have.been.calledWith('runtime.node.event_loop.utilization') - - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.max', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.min', sinon.match.number) - expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.sum', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.avg', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.median', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.95percentile', sinon.match.number) - expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.count', sinon.match.number) - - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.max', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.min', sinon.match.number) - expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.by.type.sum', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.avg', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.median', sinon.match.number) - expect(client.gauge).to.have.been.calledWith('runtime.node.gc.pause.by.type.95percentile', sinon.match.number) - expect(client.increment).to.have.been.calledWith('runtime.node.gc.pause.by.type.count', sinon.match.number) - expect(client.increment).to.have.been.calledWith( - 'runtime.node.gc.pause.by.type.count', sinon.match.any, sinon.match(val => { - return val && /^gc_type:[a-z_]+$/.test(val[0]) - }) - ) - - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.size.by.space') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.used_size.by.space') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.available_size.by.space') - expect(client.gauge).to.have.been.calledWith('runtime.node.heap.physical_size.by.space') - - expect(client.flush).to.have.been.called - - done() - } catch (e) { - done(e) - } - })) + // If a metric is leaking, it will leak expontentially because it will + // be sent one more time each flush, in addition to the previous + // flushes that also had the metric multiple times in them, so after + // 1 hour even if a single metric is leaking it would get over + // 64980 calls on its own without any other metric. A slightly lower + // value is used here to be on the safer side. + expect(client.gauge.callCount).to.be.lt(60000) + expect(client.increment.callCount).to.be.lt(60000) }) }) From 90ab0cb7de9f6c0cfa4a159bc36189b2a25408ec Mon Sep 17 00:00:00 2001 From: Yun Kim <35776586+Yun-Kim@users.noreply.github.com> Date: Tue, 25 Mar 2025 13:28:34 -0400 Subject: [PATCH 29/32] chore(llmobs): add span size telemetry metrics (#5468) * Add raw span size metric * Add processed span event size metrics * Refactor * Remove unnecessary check --- packages/dd-trace/src/llmobs/telemetry.js | 36 ++++++++++++++++++- .../dd-trace/src/llmobs/writers/spans/base.js | 11 +++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/packages/dd-trace/src/llmobs/telemetry.js b/packages/dd-trace/src/llmobs/telemetry.js index b726bb6295b..33537033c8b 100644 --- a/packages/dd-trace/src/llmobs/telemetry.js +++ b/packages/dd-trace/src/llmobs/telemetry.js @@ -18,6 +18,27 @@ const LLMObsTagger = require('./tagger') const llmobsMetrics = telemetryMetrics.manager.namespace('mlobs') +function extractIntegrationFromTags (tags) { + if (!Array.isArray(tags)) return null + const integrationTag = tags.find(tag => tag.startsWith('integration:')) + if (!integrationTag) return null + return integrationTag.split(':')[1] || null +} + +function extractTagsFromSpanEvent (event) { + const spanKind = event.meta?.['span.kind'] || '' + const integration = extractIntegrationFromTags(event.tags) + const error = event.status === 'error' + const autoinstrumented = integration != null + + return { + span_kind: spanKind, + autoinstrumented: Number(autoinstrumented), + error: error ? 1 : 0, + integration: integration || 'N/A' + } +} + function incrementLLMObsSpanStartCount (tags, value = 1) { llmobsMetrics.count('span.start', tags).inc(value) } @@ -53,7 +74,20 @@ function incrementLLMObsSpanFinishedCount (span, value = 1) { llmobsMetrics.count('span.finished', tags).inc(value) } +function recordLLMObsRawSpanSize (event, rawEventSize) { + const tags = extractTagsFromSpanEvent(event) + llmobsMetrics.distribution('span.raw_size', tags).track(rawEventSize) +} + +function recordLLMObsSpanSize (event, eventSize, shouldTruncate) { + const tags = extractTagsFromSpanEvent(event) + tags.truncated = Number(shouldTruncate) + llmobsMetrics.distribution('span.size', tags).track(eventSize) +} + module.exports = { incrementLLMObsSpanStartCount, - incrementLLMObsSpanFinishedCount + incrementLLMObsSpanFinishedCount, + recordLLMObsRawSpanSize, + recordLLMObsSpanSize } diff --git a/packages/dd-trace/src/llmobs/writers/spans/base.js b/packages/dd-trace/src/llmobs/writers/spans/base.js index e2ac1dfd751..b8575d1a010 100644 --- a/packages/dd-trace/src/llmobs/writers/spans/base.js +++ b/packages/dd-trace/src/llmobs/writers/spans/base.js @@ -4,6 +4,7 @@ const { EVP_EVENT_SIZE_LIMIT, EVP_PAYLOAD_SIZE_LIMIT } = require('../../constant const { DROPPED_VALUE_TEXT } = require('../../constants/text') const { DROPPED_IO_COLLECTION_ERROR } = require('../../constants/tags') const BaseWriter = require('../base') +const telemetry = require('../../telemetry') const logger = require('../../../log') const tracerVersion = require('../../../../../../package.json').version @@ -18,11 +19,19 @@ class LLMObsSpanWriter extends BaseWriter { append (event) { const eventSizeBytes = Buffer.from(JSON.stringify(event)).byteLength - if (eventSizeBytes > EVP_EVENT_SIZE_LIMIT) { + telemetry.recordLLMObsRawSpanSize(event, eventSizeBytes) + + const shouldTruncate = eventSizeBytes > EVP_EVENT_SIZE_LIMIT + let processedEventSizeBytes = eventSizeBytes + + if (shouldTruncate) { logger.warn(`Dropping event input/output because its size (${eventSizeBytes}) exceeds the 1MB event size limit`) event = this._truncateSpanEvent(event) + processedEventSizeBytes = Buffer.from(JSON.stringify(event)).byteLength } + telemetry.recordLLMObsSpanSize(event, processedEventSizeBytes, shouldTruncate) + if (this._bufferSize + eventSizeBytes > EVP_PAYLOAD_SIZE_LIMIT) { logger.debug('Flusing queue because queing next event will exceed EvP payload limit') this.flush() From 1b8d583a752caed80a20531f6c672ad9bb8b9b56 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 26 Mar 2025 00:16:53 +0100 Subject: [PATCH 30/32] Add a few TODO's (#5477) --- packages/datadog-instrumentations/src/router.js | 1 + packages/dd-trace/src/plugins/plugin.js | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packages/datadog-instrumentations/src/router.js b/packages/datadog-instrumentations/src/router.js index bc9ff6152e5..fb4dc57c045 100644 --- a/packages/datadog-instrumentations/src/router.js +++ b/packages/datadog-instrumentations/src/router.js @@ -5,6 +5,7 @@ const pathToRegExp = require('path-to-regexp') const shimmer = require('../../datadog-shimmer') const { addHook, channel } = require('./helpers/instrument') +// TODO: Move this function to a shared file between Express and Router function createWrapRouterMethod (name) { const enterChannel = channel(`apm:${name}:middleware:enter`) const exitChannel = channel(`apm:${name}:middleware:exit`) diff --git a/packages/dd-trace/src/plugins/plugin.js b/packages/dd-trace/src/plugins/plugin.js index 9d39320747d..cf88883b60f 100644 --- a/packages/dd-trace/src/plugins/plugin.js +++ b/packages/dd-trace/src/plugins/plugin.js @@ -18,10 +18,12 @@ class Subscription { } enable () { + // TODO: Once Node.js v18.6.0 is no longer supported, we should use `dc.subscribe(event, handler)` instead this._channel.subscribe(this._handler) } disable () { + // TODO: Once Node.js v18.6.0 is no longer supported, we should use `dc.unsubscribe(event, handler)` instead this._channel.unsubscribe(this._handler) } } From 8370d311e7d940d1c66bf141ecc7964a4c843904 Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Wed, 26 Mar 2025 09:45:03 +0100 Subject: [PATCH 31/32] Fix iast flaky code injection tests (#5460) --- .../test/appsec/iast/code_injection.integration.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dd-trace/test/appsec/iast/code_injection.integration.spec.js b/packages/dd-trace/test/appsec/iast/code_injection.integration.spec.js index 1ee892d25cd..18b6829e2d7 100644 --- a/packages/dd-trace/test/appsec/iast/code_injection.integration.spec.js +++ b/packages/dd-trace/test/appsec/iast/code_injection.integration.spec.js @@ -10,7 +10,7 @@ describe('IAST - code_injection - integration', () => { let axios, sandbox, cwd, appPort, agent, proc before(async function () { - this.timeout(process.platform === 'win32' ? 100000 : 30000) + this.timeout(process.platform === 'win32' ? 300000 : 30000) sandbox = await createSandbox( ['express'], From 38b037233cbf2d994a792c01c722450210826659 Mon Sep 17 00:00:00 2001 From: rochdev Date: Wed, 26 Mar 2025 10:08:33 -0400 Subject: [PATCH 32/32] v5.44.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8d645c06e74..00f184ba3c9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dd-trace", - "version": "5.43.0", + "version": "5.44.0", "description": "Datadog APM tracing client for JavaScript", "main": "index.js", "typings": "index.d.ts",