Skip to content

Commit 3296eb8

Browse files
[test optimization] Add dynamic instrumentation support for cucumber (#4956)
1 parent 844d623 commit 3296eb8

File tree

8 files changed

+311
-11
lines changed

8 files changed

+311
-11
lines changed

integration-tests/ci-visibility-intake.js

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ const DEFAULT_SUITES_TO_SKIP = []
2525
const DEFAULT_GIT_UPLOAD_STATUS = 200
2626
const DEFAULT_KNOWN_TESTS_UPLOAD_STATUS = 200
2727
const DEFAULT_INFO_RESPONSE = {
28-
endpoints: ['/evp_proxy/v2']
28+
endpoints: ['/evp_proxy/v2', '/debugger/v1/input']
2929
}
3030
const DEFAULT_CORRELATION_ID = '1234'
3131
const DEFAULT_KNOWN_TESTS = ['test-suite1.js.test-name1', 'test-suite2.js.test-name2']
@@ -208,7 +208,10 @@ class FakeCiVisIntake extends FakeAgent {
208208
})
209209
})
210210

211-
app.post('/api/v2/logs', express.json(), (req, res) => {
211+
app.post([
212+
'/api/v2/logs',
213+
'/debugger/v1/input'
214+
], express.json(), (req, res) => {
212215
res.status(200).send('OK')
213216
this.emit('message', {
214217
headers: req.headers,
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
const assert = require('assert')
2+
const { When, Then } = require('@cucumber/cucumber')
3+
const sum = require('./sum')
4+
5+
let count = 0
6+
7+
When('the greeter says hello', function () {
8+
this.whatIHeard = 'hello'
9+
})
10+
11+
Then('I should have heard {string}', function (expectedResponse) {
12+
sum(11, 3)
13+
assert.equal(this.whatIHeard, expectedResponse)
14+
})
15+
16+
Then('I should have flakily heard {string}', function (expectedResponse) {
17+
const shouldFail = count++ < 1
18+
if (shouldFail) {
19+
sum(11, 3)
20+
} else {
21+
sum(1, 3) // does not hit the breakpoint the second time
22+
}
23+
assert.equal(this.whatIHeard, expectedResponse)
24+
})
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
function funSum (a, b) {
2+
const localVariable = 2
3+
if (a > 10) {
4+
throw new Error('the number is too big')
5+
}
6+
7+
return a + b + localVariable
8+
}
9+
10+
module.exports = funSum
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
Feature: Greeting
3+
4+
Scenario: Say hello
5+
When the greeter says hello
6+
Then I should have heard "hello"
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
Feature: Greeting
3+
4+
Scenario: Say hello
5+
When the greeter says hello
6+
Then I should have flakily heard "hello"

integration-tests/cucumber/cucumber.spec.js

Lines changed: 192 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,11 @@ const {
3737
TEST_SUITE,
3838
TEST_CODE_OWNERS,
3939
TEST_SESSION_NAME,
40-
TEST_LEVEL_EVENT_TYPES
40+
TEST_LEVEL_EVENT_TYPES,
41+
DI_ERROR_DEBUG_INFO_CAPTURED,
42+
DI_DEBUG_ERROR_FILE,
43+
DI_DEBUG_ERROR_SNAPSHOT_ID,
44+
DI_DEBUG_ERROR_LINE
4145
} = require('../../packages/dd-trace/src/plugins/util/test')
4246
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
4347

@@ -86,10 +90,11 @@ versions.forEach(version => {
8690

8791
reportMethods.forEach((reportMethod) => {
8892
context(`reporting via ${reportMethod}`, () => {
89-
let envVars, isAgentless
93+
let envVars, isAgentless, logsEndpoint
9094
beforeEach(() => {
9195
isAgentless = reportMethod === 'agentless'
9296
envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port)
97+
logsEndpoint = isAgentless ? '/api/v2/logs' : '/debugger/v1/input'
9398
})
9499
const runModes = ['serial']
95100

@@ -1536,6 +1541,191 @@ versions.forEach(version => {
15361541
})
15371542
})
15381543
})
1544+
// Dynamic instrumentation only supported from >=8.0.0
1545+
context('dynamic instrumentation', () => {
1546+
it('does not activate if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => {
1547+
const eventsPromise = receiver
1548+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
1549+
const events = payloads.flatMap(({ payload }) => payload.events)
1550+
1551+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1552+
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1553+
1554+
assert.equal(retriedTests.length, 1)
1555+
const [retriedTest] = retriedTests
1556+
1557+
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
1558+
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
1559+
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
1560+
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
1561+
})
1562+
const logsPromise = receiver
1563+
.gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
1564+
if (payloads.length > 0) {
1565+
throw new Error('Unexpected logs')
1566+
}
1567+
}, 5000)
1568+
1569+
childProcess = exec(
1570+
'./node_modules/.bin/cucumber-js ci-visibility/features-di/test-hit-breakpoint.feature --retry 1',
1571+
{
1572+
cwd,
1573+
env: envVars,
1574+
stdio: 'pipe'
1575+
}
1576+
)
1577+
1578+
childProcess.on('exit', () => {
1579+
Promise.all([eventsPromise, logsPromise]).then(() => {
1580+
done()
1581+
}).catch(done)
1582+
})
1583+
})
1584+
1585+
it('runs retries with dynamic instrumentation', (done) => {
1586+
receiver.setSettings({
1587+
itr_enabled: false,
1588+
code_coverage: false,
1589+
tests_skipping: false,
1590+
early_flake_detection: {
1591+
enabled: false
1592+
},
1593+
flaky_test_retries_enabled: false
1594+
})
1595+
let snapshotIdByTest, snapshotIdByLog
1596+
let spanIdByTest, spanIdByLog, traceIdByTest, traceIdByLog
1597+
1598+
const eventsPromise = receiver
1599+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
1600+
const events = payloads.flatMap(({ payload }) => payload.events)
1601+
1602+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1603+
1604+
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1605+
1606+
assert.equal(retriedTests.length, 1)
1607+
const [retriedTest] = retriedTests
1608+
1609+
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
1610+
assert.propertyVal(
1611+
retriedTest.meta,
1612+
DI_DEBUG_ERROR_FILE,
1613+
'ci-visibility/features-di/support/sum.js'
1614+
)
1615+
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
1616+
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
1617+
1618+
snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
1619+
spanIdByTest = retriedTest.span_id.toString()
1620+
traceIdByTest = retriedTest.trace_id.toString()
1621+
})
1622+
1623+
const logsPromise = receiver
1624+
.gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
1625+
const [{ logMessage: [diLog] }] = payloads
1626+
assert.deepInclude(diLog, {
1627+
ddsource: 'dd_debugger',
1628+
level: 'error'
1629+
})
1630+
assert.equal(diLog.debugger.snapshot.language, 'javascript')
1631+
assert.deepInclude(diLog.debugger.snapshot.captures.lines['4'].locals, {
1632+
a: {
1633+
type: 'number',
1634+
value: '11'
1635+
},
1636+
b: {
1637+
type: 'number',
1638+
value: '3'
1639+
},
1640+
localVariable: {
1641+
type: 'number',
1642+
value: '2'
1643+
}
1644+
})
1645+
spanIdByLog = diLog.dd.span_id
1646+
traceIdByLog = diLog.dd.trace_id
1647+
snapshotIdByLog = diLog.debugger.snapshot.id
1648+
})
1649+
1650+
childProcess = exec(
1651+
'./node_modules/.bin/cucumber-js ci-visibility/features-di/test-hit-breakpoint.feature --retry 1',
1652+
{
1653+
cwd,
1654+
env: {
1655+
...envVars,
1656+
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true'
1657+
},
1658+
stdio: 'pipe'
1659+
}
1660+
)
1661+
1662+
childProcess.on('exit', () => {
1663+
Promise.all([eventsPromise, logsPromise]).then(() => {
1664+
assert.equal(snapshotIdByTest, snapshotIdByLog)
1665+
assert.equal(spanIdByTest, spanIdByLog)
1666+
assert.equal(traceIdByTest, traceIdByLog)
1667+
done()
1668+
}).catch(done)
1669+
})
1670+
})
1671+
1672+
it('does not crash if the retry does not hit the breakpoint', (done) => {
1673+
receiver.setSettings({
1674+
itr_enabled: false,
1675+
code_coverage: false,
1676+
tests_skipping: false,
1677+
early_flake_detection: {
1678+
enabled: false
1679+
},
1680+
flaky_test_retries_enabled: false
1681+
})
1682+
1683+
const eventsPromise = receiver
1684+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
1685+
const events = payloads.flatMap(({ payload }) => payload.events)
1686+
1687+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1688+
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1689+
1690+
assert.equal(retriedTests.length, 1)
1691+
const [retriedTest] = retriedTests
1692+
1693+
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
1694+
assert.propertyVal(
1695+
retriedTest.meta,
1696+
DI_DEBUG_ERROR_FILE,
1697+
'ci-visibility/features-di/support/sum.js'
1698+
)
1699+
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
1700+
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
1701+
})
1702+
const logsPromise = receiver
1703+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
1704+
if (payloads.length > 0) {
1705+
throw new Error('Unexpected logs')
1706+
}
1707+
}, 5000)
1708+
1709+
childProcess = exec(
1710+
'./node_modules/.bin/cucumber-js ci-visibility/features-di/test-not-hit-breakpoint.feature --retry 1',
1711+
{
1712+
cwd,
1713+
env: {
1714+
...envVars,
1715+
DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true'
1716+
},
1717+
stdio: 'pipe'
1718+
}
1719+
)
1720+
1721+
childProcess.on('exit', (exitCode) => {
1722+
Promise.all([eventsPromise, logsPromise]).then(() => {
1723+
assert.equal(exitCode, 0)
1724+
done()
1725+
}).catch(done)
1726+
})
1727+
})
1728+
})
15391729
}
15401730
})
15411731
})

packages/datadog-instrumentations/src/cucumber.js

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,20 @@ function getTestStatusFromRetries (testStatuses) {
126126
return 'pass'
127127
}
128128

129+
function getErrorFromCucumberResult (cucumberResult) {
130+
if (!cucumberResult.message) {
131+
return
132+
}
133+
134+
const [message] = cucumberResult.message.split('\n')
135+
const error = new Error(message)
136+
if (cucumberResult.exception) {
137+
error.type = cucumberResult.exception.type
138+
}
139+
error.stack = cucumberResult.message
140+
return error
141+
}
142+
129143
function getChannelPromise (channelToPublishTo) {
130144
return new Promise(resolve => {
131145
sessionAsyncResource.runInAsyncScope(() => {
@@ -230,9 +244,19 @@ function wrapRun (pl, isLatestVersion) {
230244
if (testCase?.testCaseFinished) {
231245
const { testCaseFinished: { willBeRetried } } = testCase
232246
if (willBeRetried) { // test case failed and will be retried
247+
let error
248+
try {
249+
const cucumberResult = this.getWorstStepResult()
250+
error = getErrorFromCucumberResult(cucumberResult)
251+
} catch (e) {
252+
// ignore error
253+
}
254+
233255
const failedAttemptAsyncResource = numAttemptToAsyncResource.get(numAttempt)
256+
const isRetry = numAttempt++ > 0
234257
failedAttemptAsyncResource.runInAsyncScope(() => {
235-
testRetryCh.publish(numAttempt++ > 0) // the current span will be finished and a new one will be created
258+
// the current span will be finished and a new one will be created
259+
testRetryCh.publish({ isRetry, error })
236260
})
237261

238262
const newAsyncResource = new AsyncResource('bound-anonymous-fn')
@@ -251,7 +275,7 @@ function wrapRun (pl, isLatestVersion) {
251275
})
252276
promise.finally(() => {
253277
const result = this.getWorstStepResult()
254-
const { status, skipReason, errorMessage } = isLatestVersion
278+
const { status, skipReason } = isLatestVersion
255279
? getStatusFromResultLatest(result)
256280
: getStatusFromResult(result)
257281

@@ -270,8 +294,10 @@ function wrapRun (pl, isLatestVersion) {
270294
}
271295
const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt)
272296

297+
const error = getErrorFromCucumberResult(result)
298+
273299
attemptAsyncResource.runInAsyncScope(() => {
274-
testFinishCh.publish({ status, skipReason, errorMessage, isNew, isEfdRetry, isFlakyRetry: numAttempt > 0 })
300+
testFinishCh.publish({ status, skipReason, error, isNew, isEfdRetry, isFlakyRetry: numAttempt > 0 })
275301
})
276302
})
277303
return promise

0 commit comments

Comments
 (0)