[test optimization] Add dynamic instrumentation support for cucumber (#4956)

Author: juan-fernandez

integration-tests/ci-visibility-intake.js

@@ -25,7 +25,7 @@ const DEFAULT_SUITES_TO_SKIP = []
 const DEFAULT_GIT_UPLOAD_STATUS = 200
 const DEFAULT_KNOWN_TESTS_UPLOAD_STATUS = 200
 const DEFAULT_INFO_RESPONSE = {
-  endpoints: ['/evp_proxy/v2']
+  endpoints: ['/evp_proxy/v2', '/debugger/v1/input']
 }
 const DEFAULT_CORRELATION_ID = '1234'
 const DEFAULT_KNOWN_TESTS = ['test-suite1.js.test-name1', 'test-suite2.js.test-name2']
@@ -208,7 +208,10 @@ class FakeCiVisIntake extends FakeAgent {
       })
     })
 
-    app.post('/api/v2/logs', express.json(), (req, res) => {
+    app.post([
+      '/api/v2/logs',
+      '/debugger/v1/input'
+    ], express.json(), (req, res) => {
       res.status(200).send('OK')
       this.emit('message', {
         headers: req.headers,

integration-tests/ci-visibility/features-di/support/steps.js

@@ -0,0 +1,24 @@
+const assert = require('assert')
+const { When, Then } = require('@cucumber/cucumber')
+const sum = require('./sum')
+
+let count = 0
+
+When('the greeter says hello', function () {
+  this.whatIHeard = 'hello'
+})
+
+Then('I should have heard {string}', function (expectedResponse) {
+  sum(11, 3)
+  assert.equal(this.whatIHeard, expectedResponse)
+})
+
+Then('I should have flakily heard {string}', function (expectedResponse) {
+  const shouldFail = count++ < 1
+  if (shouldFail) {
+    sum(11, 3)
+  } else {
+    sum(1, 3) // does not hit the breakpoint the second time
+  }
+  assert.equal(this.whatIHeard, expectedResponse)
+})

integration-tests/ci-visibility/features-di/support/sum.js

@@ -0,0 +1,10 @@
+function funSum (a, b) {
+  const localVariable = 2
+  if (a > 10) {
+    throw new Error('the number is too big')
+  }
+
+  return a + b + localVariable
+}
+
+module.exports = funSum

integration-tests/ci-visibility/features-di/test-hit-breakpoint.feature

@@ -0,0 +1,6 @@
+
+Feature: Greeting
+
+  Scenario: Say hello
+    When the greeter says hello
+    Then I should have heard "hello"

integration-tests/ci-visibility/features-di/test-not-hit-breakpoint.feature

@@ -0,0 +1,6 @@
+
+Feature: Greeting
+
+  Scenario: Say hello
+    When the greeter says hello
+    Then I should have flakily heard "hello"

integration-tests/cucumber/cucumber.spec.js

@@ -37,7 +37,11 @@ const {
   TEST_SUITE,
   TEST_CODE_OWNERS,
   TEST_SESSION_NAME,
-  TEST_LEVEL_EVENT_TYPES
+  TEST_LEVEL_EVENT_TYPES,
+  DI_ERROR_DEBUG_INFO_CAPTURED,
+  DI_DEBUG_ERROR_FILE,
+  DI_DEBUG_ERROR_SNAPSHOT_ID,
+  DI_DEBUG_ERROR_LINE
 } = require('../../packages/dd-trace/src/plugins/util/test')
 const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
 
@@ -86,10 +90,11 @@ versions.forEach(version => {
 
     reportMethods.forEach((reportMethod) => {
       context(`reporting via ${reportMethod}`, () => {
-        let envVars, isAgentless
+        let envVars, isAgentless, logsEndpoint
         beforeEach(() => {
           isAgentless = reportMethod === 'agentless'
           envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port)
+          logsEndpoint = isAgentless ? '/api/v2/logs' : '/debugger/v1/input'
         })
         const runModes = ['serial']
 
@@ -1536,6 +1541,191 @@ versions.forEach(version => {
               })
             })
           })
+          // Dynamic instrumentation only supported from >=8.0.0
+          context('dynamic instrumentation', () => {
+            it('does not activate if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is not set', (done) => {
+              const eventsPromise = receiver
+                .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+                  const events = payloads.flatMap(({ payload }) => payload.events)
+
+                  const tests = events.filter(event => event.type === 'test').map(event => event.content)
+                  const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
+
+                  assert.equal(retriedTests.length, 1)
+                  const [retriedTest] = retriedTests
+
+                  assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
+                  assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
+                  assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
+                  assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
+                })
+              const logsPromise = receiver
+                .gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
+                  if (payloads.length > 0) {
+                    throw new Error('Unexpected logs')
+                  }
+                }, 5000)
+
+              childProcess = exec(
+                './node_modules/.bin/cucumber-js ci-visibility/features-di/test-hit-breakpoint.feature --retry 1',
+                {
+                  cwd,
+                  env: envVars,
+                  stdio: 'pipe'
+                }
+              )
+
+              childProcess.on('exit', () => {
+                Promise.all([eventsPromise, logsPromise]).then(() => {
+                  done()
+                }).catch(done)
+              })
+            })
+
+            it('runs retries with dynamic instrumentation', (done) => {
+              receiver.setSettings({
+                itr_enabled: false,
+                code_coverage: false,
+                tests_skipping: false,
+                early_flake_detection: {
+                  enabled: false
+                },
+                flaky_test_retries_enabled: false
+              })
+              let snapshotIdByTest, snapshotIdByLog
+              let spanIdByTest, spanIdByLog, traceIdByTest, traceIdByLog
+
+              const eventsPromise = receiver
+                .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
+                  const events = payloads.flatMap(({ payload }) => payload.events)
+
+                  const tests = events.filter(event => event.type === 'test').map(event => event.content)
+
+                  const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
+
+                  assert.equal(retriedTests.length, 1)
+                  const [retriedTest] = retriedTests
+
+                  assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
+                  assert.propertyVal(
+                    retriedTest.meta,
+                    DI_DEBUG_ERROR_FILE,
+                    'ci-visibility/features-di/support/sum.js'
+                  )
+                  assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
+                  assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
+
+                  snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
+                  spanIdByTest = retriedTest.span_id.toString()
+                  traceIdByTest = retriedTest.trace_id.toString()
+                })
+
+              const logsPromise = receiver
+                .gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
+                  const [{ logMessage: [diLog] }] = payloads
+                  assert.deepInclude(diLog, {
+                    ddsource: 'dd_debugger',
+                    level: 'error'
+                  })
+                  assert.equal(diLog.debugger.snapshot.language, 'javascript')
+                  assert.deepInclude(diLog.debugger.snapshot.captures.lines['4'].locals, {
+                    a: {
+                      type: 'number',
+                      value: '11'
+                    },
+                    b: {
+                      type: 'number',
+                      value: '3'
+                    },
+                    localVariable: {
+                      type: 'number',
+                      value: '2'
+                    }
+                  })
+                  spanIdByLog = diLog.dd.span_id
+                  traceIdByLog = diLog.dd.trace_id
+                  snapshotIdByLog = diLog.debugger.snapshot.id
+                })
+
+              childProcess = exec(
+                './node_modules/.bin/cucumber-js ci-visibility/features-di/test-hit-breakpoint.feature --retry 1',
+                {
+                  cwd,
+                  env: {
+                    ...envVars,
+                    DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true'
+                  },
+                  stdio: 'pipe'
+                }
+              )
+
+              childProcess.on('exit', () => {
+                Promise.all([eventsPromise, logsPromise]).then(() => {
+                  assert.equal(snapshotIdByTest, snapshotIdByLog)
+                  assert.equal(spanIdByTest, spanIdByLog)
+                  assert.equal(traceIdByTest, traceIdByLog)
+                  done()
+                }).catch(done)
+              })
+            })
+
+            it('does not crash if the retry does not hit the breakpoint', (done) => {
+              receiver.setSettings({
+                itr_enabled: false,
+                code_coverage: false,
+                tests_skipping: false,
+                early_flake_detection: {
+                  enabled: false
+                },
+                flaky_test_retries_enabled: false
+              })
+
+              const eventsPromise = receiver
+                .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+                  const events = payloads.flatMap(({ payload }) => payload.events)
+
+                  const tests = events.filter(event => event.type === 'test').map(event => event.content)
+                  const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
+
+                  assert.equal(retriedTests.length, 1)
+                  const [retriedTest] = retriedTests
+
+                  assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
+                  assert.propertyVal(
+                    retriedTest.meta,
+                    DI_DEBUG_ERROR_FILE,
+                    'ci-visibility/features-di/support/sum.js'
+                  )
+                  assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
+                  assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
+                })
+              const logsPromise = receiver
+                .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
+                  if (payloads.length > 0) {
+                    throw new Error('Unexpected logs')
+                  }
+                }, 5000)
+
+              childProcess = exec(
+                './node_modules/.bin/cucumber-js ci-visibility/features-di/test-not-hit-breakpoint.feature --retry 1',
+                {
+                  cwd,
+                  env: {
+                    ...envVars,
+                    DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED: 'true'
+                  },
+                  stdio: 'pipe'
+                }
+              )
+
+              childProcess.on('exit', (exitCode) => {
+                Promise.all([eventsPromise, logsPromise]).then(() => {
+                  assert.equal(exitCode, 0)
+                  done()
+                }).catch(done)
+              })
+            })
+          })
         }
       })
     })

packages/datadog-instrumentations/src/cucumber.js

@@ -126,6 +126,20 @@ function getTestStatusFromRetries (testStatuses) {
   return 'pass'
 }
 
+function getErrorFromCucumberResult (cucumberResult) {
+  if (!cucumberResult.message) {
+    return
+  }
+
+  const [message] = cucumberResult.message.split('\n')
+  const error = new Error(message)
+  if (cucumberResult.exception) {
+    error.type = cucumberResult.exception.type
+  }
+  error.stack = cucumberResult.message
+  return error
+}
+
 function getChannelPromise (channelToPublishTo) {
   return new Promise(resolve => {
     sessionAsyncResource.runInAsyncScope(() => {
@@ -230,9 +244,19 @@ function wrapRun (pl, isLatestVersion) {
         if (testCase?.testCaseFinished) {
           const { testCaseFinished: { willBeRetried } } = testCase
           if (willBeRetried) { // test case failed and will be retried
+            let error
+            try {
+              const cucumberResult = this.getWorstStepResult()
+              error = getErrorFromCucumberResult(cucumberResult)
+            } catch (e) {
+              // ignore error
+            }
+
             const failedAttemptAsyncResource = numAttemptToAsyncResource.get(numAttempt)
+            const isRetry = numAttempt++ > 0
             failedAttemptAsyncResource.runInAsyncScope(() => {
-              testRetryCh.publish(numAttempt++ > 0) // the current span will be finished and a new one will be created
+              // the current span will be finished and a new one will be created
+              testRetryCh.publish({ isRetry, error })
             })
 
             const newAsyncResource = new AsyncResource('bound-anonymous-fn')
@@ -251,7 +275,7 @@ function wrapRun (pl, isLatestVersion) {
       })
       promise.finally(() => {
         const result = this.getWorstStepResult()
-        const { status, skipReason, errorMessage } = isLatestVersion
+        const { status, skipReason } = isLatestVersion
           ? getStatusFromResultLatest(result)
           : getStatusFromResult(result)
 
@@ -270,8 +294,10 @@ function wrapRun (pl, isLatestVersion) {
         }
         const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt)
 
+        const error = getErrorFromCucumberResult(result)
+
         attemptAsyncResource.runInAsyncScope(() => {
-          testFinishCh.publish({ status, skipReason, errorMessage, isNew, isEfdRetry, isFlakyRetry: numAttempt > 0 })
+          testFinishCh.publish({ status, skipReason, error, isNew, isEfdRetry, isFlakyRetry: numAttempt > 0 })
         })
       })
       return promise