chore: Updated versioned tests to support upcoming tooling (#2147)

newrelic · Apr 16, 2024 · 38e47d9 · 38e47d9
1 parent 63a5ddc
commit 38e47d9
Show file tree

Hide file tree

Showing 51 changed files with 183 additions and 61 deletions.
diff --git a/test/versioned/Readme.md b/test/versioned/Readme.md
@@ -0,0 +1,102 @@
+# Versioned Tests
+
+The subdirectories within this directory represent test suites that verify
+our agent compatibility across multiple versions of our supported modules.
+When these tests are run, through our versioned test runner tool, each suite
+is run multiple times utilizing different versions of the module that satisfy
+configured version constraints.
+
+For example, if a supported module has releases `1.0.0` through `1.10.0`, with
+a version for each minor between `0` and `10`, then our versioned test runner
+will run the test suite across a sampling of versions in that range, e.g.
+versions `1.0.0`, `1.3.0`, and `1.10.0`.
+
+## Versioned Tests `package.json`
+
+The versioned test runner reads a `package.json` in each test suite. This
+`package.json` describes the constraints for the suite and has a few properties
+that are specific to our versioned test runner. The following is a
+[jsonc](https://en.wikipedia.org/wiki/JSON#JSONC) representation of such a
+`package.json` that describes the unique properties:
+
+```jsonc
+{
+  // `name` is typically ignored.
+  "name": "module-tests",
+
+  // `targets` indicates which modules are being verified by the test suite.
+  // This is utilized by tooling to build a compatibility document. There _must_
+  // be at least one value that matches a dependency name in a subsequent
+  // `tests` block. If this suite is verifying the compatibility of multiple
+  // modules, include all such module names in the array.
+  //
+  // If this property is omitted, the tool used to build our compatibility
+  // document(s) will not include any modules from this versioned test suite.
+  "targets": ["module"],
+
+  // `version` is ignored.
+  "version": "0.0.0",
+
+  // `private` should be set to `true`.
+  "private": true,
+
+  // `tests` contains blocks that describe the tests the versioned test runner
+  // should run for this suite and under what constraints. Each block will
+  // result in at least one test run by the test runner.
+  "tests": [
+    {
+      // `engines` is a typical package.json engines block.
+      "engines": {
+        // `node` indicates which versions of Node.js should be used to run
+        // this test block. Typically, a basic `>=` qualifier will be used, but
+        // a static version is also likely. If the version of Node.js being
+        // used to run the suite does not match the contraint, then the test
+        // block will be skipped.
+        "node": ">= 18"
+      },
+
+      // `dependencies` lists dependencies that a needed in order to execute
+      // the test block. In most cases, only the module under test will be
+      // present.
+      "dependencies": {
+        // For the dependency named "module-name", run the suite with samples
+        // from the provided semver range (https://docs.npmjs.com/cli/v6/using-npm/semver#advanced-range-syntax).
+        //
+        // The minimum version across all test blocks will be utilized to
+        // indicate the minimum supported version of a module by our agent
+        // if that module name is listed in the top-level `targets` property.
+        //
+        // Note: this may also be an object with a special format. See the next
+        // example block.
+        "module-name": ">=1.0.0 <2.0.0"
+      },
+
+      // `files` lists out the test files that comprise the test suite for the
+      // current block.
+      "files": [
+        "test-one.tap.js",
+        "test-two.tap.js"
+      ]
+    },
+
+    // This example block will only run on Node.js 20.x. Pay attention
+    // to the "dependencies" section for a special depedency declaration
+    // supported by our versioned test runner.
+    {
+      "engines": { "node": "20" },
+      "dependencies": {
+        "module-name": {
+          // Again, a standard semver range to indicate the versions of the
+          // module to sample from.
+          "versions": ">=1.0.0 <2.0.0",
+
+          // How many samples, across the provided versions range, to conduct
+          // when testing with this test block. It should be a string value,
+          // although some of our tooling will likely coerce it to an integer.
+          "samples": "2"
+        }
+      }
+    }
+  ]
+}
+```
diff --git a/test/versioned/amqplib/package.json b/test/versioned/amqplib/package.json
@@ -1,5 +1,6 @@
 {
   "name": "ampqlib-tests",
+  "targets": ["amqplib"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/bluebird/package.json b/test/versioned/bluebird/package.json
@@ -1,5 +1,6 @@
 {
   "name": "bluebird-tests",
+  "targets": ["bluebird"],
   "version": "0.0.0",
   "private": true,
   "tests": [
@@ -8,7 +9,7 @@
         "node": ">=16"
       },
       "dependencies": {
-        "bluebird": ">=2"
+        "bluebird": ">=2.0.0"
       },
       "files": [
         "regressions.tap.js",
@@ -20,7 +21,7 @@
         "node": ">=16"
       },
       "dependencies": {
-        "bluebird": ">=3"
+        "bluebird": ">=3.0.0"
       },
       "files": [
         "methods.tap.js"

diff --git a/test/versioned/bunyan/package.json b/test/versioned/bunyan/package.json
@@ -1,5 +1,6 @@
 {
   "name": "bunyan-tests",
+  "targets": ["bunyan"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/cassandra-driver/package.json b/test/versioned/cassandra-driver/package.json
@@ -1,5 +1,6 @@
 {
   "name": "cassandra-driver-tests",
+  "targets": ["cassandra-driver"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/connect/package.json b/test/versioned/connect/package.json
@@ -1,5 +1,6 @@
 {
   "name": "connect-tests",
+  "targets": ["connect"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/director/package.json b/test/versioned/director/package.json
@@ -1,5 +1,6 @@
 {
   "name": "director-tests",
+  "targets": ["director"],
   "version": "0.0.0",
   "private": true,
   "tests": [
@@ -8,7 +9,7 @@
         "node": ">=16"
       },
       "dependencies": {
-        "director": ">=1.2",
+        "director": ">=1.2.0",
         "express": "4.16"
       },
       "files": [

diff --git a/test/versioned/elastic/package.json b/test/versioned/elastic/package.json
@@ -1,12 +1,15 @@
 {
   "name": "elasticsearch-tests",
+  "targets": ["@elastic/elasticsearch"],
   "version": "0.0.0",
   "private": true,
   "engines": {
     "node": ">=16"
   },
   "tests": [
     {
+      "supported": false,
+      "comment": "Used to assert our instrumentation does not get loaded on old versions.",
       "engines": {
         "node": ">=16"
       },

diff --git a/test/versioned/express-esm/package.json b/test/versioned/express-esm/package.json
@@ -1,5 +1,6 @@
 {
   "name": "express-esm-tests",
+  "targets": ["express"],
   "version": "0.0.0",
   "type": "module",
   "private": true,

diff --git a/test/versioned/express/package.json b/test/versioned/express/package.json
@@ -1,5 +1,6 @@
 {
   "name": "express-tests",
+  "targets": ["express"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/fastify/package.json b/test/versioned/fastify/package.json
@@ -1,5 +1,6 @@
 {
   "name": "fastify-tests",
+  "targets": ["fastify"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/generic-pool/package.json b/test/versioned/generic-pool/package.json
@@ -1,5 +1,6 @@
 {
   "name": "generic-pool-tests",
+  "targets": ["generic-pool"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/grpc-esm/package.json b/test/versioned/grpc-esm/package.json
@@ -1,5 +1,6 @@
 {
   "name": "grpc-esm-tests",
+  "targets": ["@grpc/grpc-js"],
   "version": "0.0.0",
   "type": "module",
   "private": true,

diff --git a/test/versioned/grpc/package.json b/test/versioned/grpc/package.json
@@ -1,5 +1,6 @@
 {
   "name": "grpc-tests",
+  "targets": ["@grpc/grpc-js"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/hapi/package.json b/test/versioned/hapi/package.json
@@ -1,5 +1,6 @@
 {
   "name": "hapi-tests",
+  "targets": ["@hapi/hapi"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/ioredis/package.json b/test/versioned/ioredis/package.json
@@ -1,5 +1,6 @@
 {
   "name": "ioredis-test",
+  "targets": ["ioredis"],
   "version": "0.0.0",
   "private": true,
   "tests": [
@@ -19,7 +20,7 @@
         "node": ">=16"
       },
       "dependencies": {
-        "ioredis": ">=4"
+        "ioredis": ">=4.0.0"
       },
       "files": [
         "ioredis.tap.js"

diff --git a/test/versioned/langchain/package.json b/test/versioned/langchain/package.json
@@ -1,5 +1,6 @@
 {
   "name": "langchain-tests",
+  "targets": ["@langchain/core"],
   "version": "0.0.0",
   "private": true,
   "engines": {

diff --git a/test/versioned/mongodb-esm/package.json b/test/versioned/mongodb-esm/package.json
@@ -1,5 +1,6 @@
 {
   "name": "mongodb-esm-tests",
+  "targets": ["mongodb"],
   "version": "0.0.0",
   "type": "module",
   "private": true,

diff --git a/test/versioned/mongodb/package.json b/test/versioned/mongodb/package.json
@@ -1,5 +1,6 @@
 {
   "name": "mongodb-tests",
+  "targets": ["mongodb"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/mysql/package.json b/test/versioned/mysql/package.json
@@ -1,5 +1,6 @@
 {
   "name": "mysql-tests",
+  "targets": ["mysql"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/mysql2/package.json b/test/versioned/mysql2/package.json
@@ -1,5 +1,6 @@
 {
   "name": "mysql2-tests",
+  "targets": ["mysql2"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/nestjs/package.json b/test/versioned/nestjs/package.json
@@ -1,5 +1,6 @@
 {
   "name": "nestjs-tests",
+  "targets": ["@nestjs/cli"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/openai/package.json b/test/versioned/openai/package.json
@@ -1,5 +1,6 @@
 {
   "name": "openai-tests",
+  "targets": ["openai"],
   "version": "0.0.0",
   "private": true,
   "engines": {

diff --git a/test/versioned/pg-esm/package.json b/test/versioned/pg-esm/package.json
@@ -1,5 +1,6 @@
 {
   "name": "pg-esm-tests",
+  "targets": ["pg"],
   "type": "module",
   "version": "0.0.0",
   "private": true,
@@ -9,7 +10,7 @@
         "node": ">=16.12.0"
       },
       "dependencies": {
-        "pg": ">=8.2 <8.8",
+        "pg": ">=8.2.0 <8.8.0",
         "pg-native": ">=2"
       },
       "files": [
@@ -23,7 +24,7 @@
         "node": ">=16.12.0"
       },
       "dependencies": {
-        "pg": ">=8.8",
+        "pg": ">=8.8.0",
         "pg-native": ">=3"
       },
       "files": [

diff --git a/test/versioned/pg/package.json b/test/versioned/pg/package.json
@@ -1,5 +1,6 @@
 {
   "name": "pg-tests",
+  "targets": ["pg"],
   "version": "0.0.0",
   "private": true,
   "tests": [
@@ -8,7 +9,7 @@
         "node": ">=16"
       },
       "dependencies": {
-        "pg": ">=8.2 <8.8",
+        "pg": ">=8.2.0 <8.8.0",
         "pg-native": ">=2"
       },
       "files": [
@@ -22,7 +23,7 @@
         "node": ">=16"
       },
       "dependencies": {
-        "pg": ">=8.8",
+        "pg": ">=8.8.0",
         "pg-native": ">=3"
       },
       "files": [

diff --git a/test/versioned/pino/package.json b/test/versioned/pino/package.json
@@ -1,5 +1,6 @@
 {
   "name": "pino-tests",
+  "targets": ["pino"],
   "version": "0.0.0",
   "private": true,
   "tests": [

diff --git a/test/versioned/prisma/package.json b/test/versioned/prisma/package.json
@@ -1,5 +1,6 @@
 {
   "name": "prisma-tests",
+  "targets": ["@prisma/client"],
   "version": "0.0.0",
   "private": true,
   "engines": {

diff --git a/test/versioned/redis/package.json b/test/versioned/redis/package.json
@@ -1,5 +1,6 @@
 {
   "name": "redis-tests",
+  "targets": ["redis"],
   "version": "0.0.0",
   "private": true,
   "tests": [
@@ -41,7 +42,7 @@
         "node": ">=16"
       },
       "dependencies": {
-        "redis": ">=2.6.0 < 4"
+        "redis": ">=2.6.0 < 4.0.0"
       },
       "files": [
         "redis.tap.js"
@@ -52,7 +53,7 @@
         "node": ">=16"
       },
       "dependencies": {
-        "redis": ">=4"
+        "redis": ">=4.0.0"
       },
       "files": [
         "redis-v4.tap.js"