[DO NOT MERGE] try to fail CI by detecting non-executed tests

.github/workflows/project.yml

@@ -89,14 +89,13 @@ jobs:
       - run: yarn type:test
       - run: yarn type:doc
 
-  # TODO: Remove need for `npm show` before re-enabling to avoid rate limit errors.
-  # verify-yaml:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-  #     - uses: ./.github/actions/node/active-lts
-  #     - uses: ./.github/actions/install
-  #     - run: node scripts/verify-ci-config.js
+  verify-yaml:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: ./.github/actions/node/active-lts
+      - uses: ./.github/actions/install
+      - run: node scripts/verify-ci-config.js
 
   yarn-dedupe:
     runs-on: ubuntu-latest

packages/dd-trace/test/appsec/test.plugin.spec.js

@@ -0,0 +1,109 @@
+'use strict'
+
+const Axios = require('axios')
+const { assert } = require('chai')
+
+const agent = require('../plugins/agent')
+const appsec = require('../../src/appsec')
+const Config = require('../../src/config')
+
+const { withVersions } = require('../../dd-trace/test/setup/mocha')
+
+function assertFingerprintInTraces (traces) {
+  const span = traces[0][0]
+  assert.property(span.meta, '_dd.appsec.fp.http.header')
+  assert.equal(span.meta['_dd.appsec.fp.http.header'], 'hdr-0110000110-6431a3e6-4-c348f529')
+  assert.property(span.meta, '_dd.appsec.fp.http.network')
+  assert.equal(span.meta['_dd.appsec.fp.http.network'], 'net-0-0000000000')
+  assert.property(span.meta, '_dd.appsec.fp.http.endpoint')
+  assert.equal(span.meta['_dd.appsec.fp.http.endpoint'], 'http-post-7e93fba0--f29f6224')
+}
+
+withVersions('passport-local', 'passport-local', version => {
+  describe('Attacker fingerprinting', () => {
+    let port, server, axios
+
+    before(() => {
+      return agent.load(['express', 'http'], { client: false })
+    })
+
+    before(() => {
+      appsec.enable(new Config({
+        appsec: true
+      }))
+    })
+
+    before((done) => {
+      const express = require('../../../../versions/express').get()
+      const bodyParser = require('../../../../versions/body-parser').get()
+      const passport = require('../../../../versions/passport').get()
+      const LocalStrategy = require(`../../../../versions/passport-local@${version}`).get()
+
+      const app = express()
+      app.use(bodyParser.json())
+      app.use(passport.initialize())
+
+      passport.use(new LocalStrategy(
+        function verify (username, password, done) {
+          if (username === 'success') {
+            done(null, {
+              id: 1234,
+              username
+            })
+          } else {
+            done(null, false)
+          }
+        }
+      ))
+
+      app.post('/login', passport.authenticate('local', { session: false }), function (req, res) {
+        res.end()
+      })
+
+      server = app.listen(port, () => {
+        port = server.address().port
+        axios = Axios.create({
+          baseURL: `http://localhost:${port}`
+        })
+        done()
+      })
+    })
+
+    after(() => {
+      server.close()
+      return agent.close({ ritmReset: false })
+    })
+
+    after(() => {
+      appsec.disable()
+    })
+
+    it('should report http fingerprints on login fail', async () => {
+      try {
+        await axios.post(
+          `http://localhost:${port}/login`,
+          {
+            username: 'fail',
+            password: '1234'
+          }
+        )
+      } catch (e) {}
+
+      await agent.use(assertFingerprintInTraces)
+
+      throw new Error('CI SHOULD FAIL')
+    })
+
+    it('should report http fingerprints on login successful', async () => {
+      await axios.post(
+        `http://localhost:${port}/login`,
+        {
+          username: 'success',
+          password: '1234'
+        }
+      )
+
+      await agent.use(assertFingerprintInTraces)
+    })
+  })
+})

scripts/verify-ci-config.js

@@ -5,9 +5,9 @@
 
 const fs = require('fs')
 const path = require('path')
-const util = require('util')
+// const util = require('util')
 const yaml = require('yaml')
-const semver = require('semver')
+// const semver = require('semver')
 const { execSync } = require('child_process')
 const Module = require('module')
 const { getAllInstrumentations } = require('../packages/dd-trace/test/setup/helpers/load-inst')
@@ -29,13 +29,27 @@ if (!Module.isBuiltin) {
   Module.isBuiltin = mod => Module.builtinModules.includes(mod)
 }
 
-const nodeMajor = Number(process.versions.node.split('.')[0])
+// const nodeMajor = Number(process.versions.node.split('.')[0])
 
 const instrumentations = getAllInstrumentations()
 
-const versions = {}
+// const versions = {}
 
 const allTestedPlugins = new Set()
+const appsecTestedPlugins = new Set()
+
+function getStrayPluginFiles (folder) {
+  const pluginFiles = execSync(`find ${folder} -type f -name "*.plugin.spec.js"`)
+    .toString()
+    .split('\n')
+    .slice(0, -1)
+    .map(filepath => {
+      const split = path.basename(filepath).split('.')
+      return split[split.length - 4]
+    })
+
+  return new Set(pluginFiles)
+}
 
 function checkPlugins (yamlPath) {
   const yamlContent = yaml.parse(fs.readFileSync(yamlPath, 'utf8'))
@@ -47,9 +61,12 @@ function checkPlugins (yamlPath) {
     if (!job.env || !job.env.PLUGINS) continue
 
     const pluginName = job.env.PLUGINS
-    if (!yamlPath.includes('appsec')) {
+    if (yamlPath.includes('appsec')) {
+      pluginName.split('|').forEach(plugin => appsecTestedPlugins.add(plugin))
+    } else {
       pluginName.split('|').forEach(plugin => allTestedPlugins.add(plugin))
     }
+
     if (Module.isBuiltin(pluginName)) continue
     const rangesFromYaml = getRangesFromYaml(job)
     if (rangesFromYaml) {
@@ -63,26 +80,26 @@ function checkPlugins (yamlPath) {
     }
   }
 
-  for (const pluginName in rangesPerPluginFromYaml) {
-    const yamlRanges = Array.from(rangesPerPluginFromYaml[pluginName])
-    const instRanges = Array.from(rangesPerPluginFromInst[pluginName])
-    const yamlVersions = getMatchingVersions(pluginName, yamlRanges)
-    const instVersions = getMatchingVersions(pluginName, instRanges)
-    if (pluginName !== 'next' && !util.isDeepStrictEqual(yamlVersions, instVersions)) {
-      const opts = { colors: true }
-      const colors = x => util.inspect(x, opts)
-      pluginErrorMsg(pluginName, 'Mismatch', `
-Valid version ranges from YAML: ${colors(yamlRanges)}
-Valid version ranges from INST: ${colors(instRanges)}
-${mismatching(yamlVersions, instVersions)}
-Note that versions may be dependent on Node.js version. This is Node.js v${colors(nodeMajor)}
-
-> These don't match the same sets of versions in npm.
->
-> Please check ${yamlPath} and the instrumentations
-> for ${pluginName} to see that the version ranges match.`.trim())
-    }
-  }
+  // DISABLED FOR NOW BECAUSE NPM SHOW GETS RATE LIMITED
+  //   for (const pluginName in rangesPerPluginFromYaml) {
+  //     const yamlRanges = Array.from(rangesPerPluginFromYaml[pluginName])
+  //     const instRanges = Array.from(rangesPerPluginFromInst[pluginName])
+  //     const yamlVersions = getMatchingVersions(pluginName, yamlRanges)
+  //     const instVersions = getMatchingVersions(pluginName, instRanges)
+  //     if (pluginName !== 'next' && !util.isDeepStrictEqual(yamlVersions, instVersions)) {
+  //       const opts = { colors: true }
+  //       const colors = x => util.inspect(x, opts)
+  //       pluginErrorMsg(pluginName, 'Mismatch', `
+  // Valid version ranges from YAML: ${colors(yamlRanges)}
+  // Valid version ranges from INST: ${colors(instRanges)}
+  // ${mismatching(yamlVersions, instVersions)}
+  // Note that versions may be dependent on Node.js version. This is Node.js v${colors(nodeMajor)}
+  // > These don't match the same sets of versions in npm.
+  // >
+  // > Please check ${yamlPath} and the instrumentations
+  // > for ${pluginName} to see that the version ranges match.`.trim())
+  //     }
+  //   }
 }
 
 function getRangesFromYaml (job) {
@@ -104,26 +121,26 @@ function getRangesFromYaml (job) {
   return null
 }
 
-function getMatchingVersions (name, ranges) {
-  if (!versions[name]) {
-    versions[name] = JSON.parse(execSync('npm show ' + name + ' versions --json').toString())
-  }
-  return versions[name].filter(version => ranges.some(range => semver.satisfies(version, range)))
-}
+// function getMatchingVersions (name, ranges) {
+//   if (!versions[name]) {
+//     versions[name] = JSON.parse(execSync('npm show ' + name + ' versions --json').toString())
+//   }
+//   return versions[name].filter(version => ranges.some(range => semver.satisfies(version, range)))
+// }
 
-function mismatching (yamlVersions, instVersions) {
-  const yamlSet = new Set(yamlVersions)
-  const instSet = new Set(instVersions)
+// function mismatching (yamlVersions, instVersions) {
+//   const yamlSet = new Set(yamlVersions)
+//   const instSet = new Set(instVersions)
 
-  const onlyInYaml = yamlVersions.filter(v => !instSet.has(v))
-  const onlyInInst = instVersions.filter(v => !yamlSet.has(v))
+//   const onlyInYaml = yamlVersions.filter(v => !instSet.has(v))
+//   const onlyInInst = instVersions.filter(v => !yamlSet.has(v))
 
-  const opts = { colors: true }
-  return [
-    `Versions only in YAML: ${util.inspect(onlyInYaml, opts)}`,
-    `Versions only in INST: ${util.inspect(onlyInInst, opts)}`
-  ].join('\n')
-}
+//   const opts = { colors: true }
+//   return [
+//     `Versions only in YAML: ${util.inspect(onlyInYaml, opts)}`,
+//     `Versions only in INST: ${util.inspect(onlyInInst, opts)}`
+//   ].join('\n')
+// }
 
 function pluginErrorMsg (pluginName, title, message) {
   errorMsg(title + ' for ' + pluginName, message)
@@ -135,6 +152,7 @@ checkPlugins(path.join(__dirname, '..', '.github', 'workflows', 'appsec.yml'))
 checkPlugins(path.join(__dirname, '..', '.github', 'workflows', 'llmobs.yml'))
 checkPlugins(path.join(__dirname, '..', '.github', 'workflows', 'platform.yml'))
 checkPlugins(path.join(__dirname, '..', '.github', 'workflows', 'test-optimization.yml'))
+checkPlugins(path.join(__dirname, '..', '.github', 'workflows', 'serverless.yml'))
 {
   const testDir = path.join(__dirname, '..', 'packages', 'datadog-instrumentations', 'test')
   const testedInstrumentations = fs.readdirSync(testDir)
@@ -154,6 +172,14 @@ checkPlugins(path.join(__dirname, '..', '.github', 'workflows', 'test-optimizati
       pluginErrorMsg(plugin, 'ERROR', 'Plugin is tested but not in at least one GitHub workflow')
     }
   }
+
+  for (const plugin of getStrayPluginFiles(path.join(__dirname, '..', 'packages', 'dd-trace', 'test', 'appsec'))) {
+    if (!appsecTestedPlugins.has(plugin)) {
+      pluginErrorMsg(plugin, 'ERROR', 'Appsec plugin is tested but not in at least one GitHub workflow')
+    }
+  }
+  // TODO: do this with all jobs that have excludes in package.json
+
 }
 
 /// /