test(NODE-5181): update fle2 v2 spec tests (#3630)

Co-authored-by: Durran Jordan <durran@gmail.com>

Author: nbbeeken

.evergreen/config.yml

@@ -2448,7 +2448,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: 77b51c00ab4ff58916dd39f55657e1ecc0af281c
+          CSFLE_GIT_REF: 1524eac203e4145e9f4835e519f1e4663ff15953
   - name: run-custom-csfle-tests-5.0-master
     tags:
       - run-custom-dependency-tests
@@ -2478,7 +2478,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: 77b51c00ab4ff58916dd39f55657e1ecc0af281c
+          CSFLE_GIT_REF: 1524eac203e4145e9f4835e519f1e4663ff15953
   - name: run-custom-csfle-tests-rapid-master
     tags:
       - run-custom-dependency-tests
@@ -2508,7 +2508,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: 77b51c00ab4ff58916dd39f55657e1ecc0af281c
+          CSFLE_GIT_REF: 1524eac203e4145e9f4835e519f1e4663ff15953
   - name: run-custom-csfle-tests-latest-master
     tags:
       - run-custom-dependency-tests

.evergreen/generate_evergreen_tasks.js

@@ -603,7 +603,7 @@ const oneOffFuncAsTasks = oneOffFuncs.map(oneOffFunc => ({
   ]
 }));
 
-const FLE_PINNED_COMMIT = '77b51c00ab4ff58916dd39f55657e1ecc0af281c'
+const FLE_PINNED_COMMIT = '1524eac203e4145e9f4835e519f1e4663ff15953'
 
 for (const version of ['5.0', 'rapid', 'latest']) {
   for (const ref of [FLE_PINNED_COMMIT, 'master']) {

.evergreen/install-dependencies.sh

@@ -110,4 +110,5 @@ fi
 
 echo "npm version: $(npm -v)"
 
-npm install "${NPM_OPTIONS}"
+# TODO(NODE-5180): remove --force option
+npm install --force "${NPM_OPTIONS}"

.evergreen/run-custom-csfle-tests.sh

@@ -52,7 +52,8 @@ popd # mongo-c-driver
 
 pushd libmongocrypt/bindings/node
 
-npm install --production --ignore-scripts
+# TODO(NODE-5180): remove --force option
+npm install --force --production --ignore-scripts
 bash ./etc/build-static.sh
 
 popd # libmongocrypt/bindings/node
@@ -81,7 +82,8 @@ pushd ../csfle-deps-tmp/libmongocrypt/bindings/node
 killall mongocryptd || true
 
 # only prod deps were installed earlier, install devDependencies here (except for mongodb!)
-npm install --ignore-scripts
+# TODO(NODE-5180): remove --force option
+npm install --force --ignore-scripts
 
 # copy mongodb into CSFLE's node_modules
 rm -rf node_modules/mongodb

.evergreen/run-serverless-tests.sh

@@ -10,7 +10,8 @@ if [ -z ${MONGODB_URI+omitted} ]; then echo "MONGODB_URI is unset" && exit 1; fi
 if [ -z ${SERVERLESS_ATLAS_USER+omitted} ]; then echo "SERVERLESS_ATLAS_USER is unset" && exit 1; fi
 if [ -z ${SERVERLESS_ATLAS_PASSWORD+omitted} ]; then echo "SERVERLESS_ATLAS_PASSWORD is unset" && exit 1; fi
 
-npm install mongodb-client-encryption@"2.4.0"
+# TODO(NODE-5180): remove --force option
+npm install --force 'mongodb-client-encryption@alpha'
 
 npx mocha \
   --config test/mocha_mongodb.json \

.evergreen/run-tests.sh

@@ -52,9 +52,10 @@ else
   source "$DRIVERS_TOOLS"/.evergreen/csfle/set-temp-creds.sh
 fi
 
-npm install mongodb-client-encryption@"2.5.0"
-npm install @mongodb-js/zstd
-npm install snappy
+# TODO(NODE-5180): remove --force option
+npm install --force 'mongodb-client-encryption@alpha'
+npm install --force @mongodb-js/zstd
+npm install --force snappy
 
 export AUTH=$AUTH
 export SINGLE_MONGOS_LB_URI=${SINGLE_MONGOS_LB_URI}

test/integration/client-side-encryption/client_side_encryption.prose.22.range_explicit_encryption.test.ts

@@ -20,8 +20,9 @@ const metaData: MongoDBMetadataUI = {
   requires: {
     clientSideEncryption: true,
 
-    // The Range Explicit Encryption tests require MongoDB server 6.2+. The tests must not run against a standalone.
-    mongodb: '>=6.2.0',
+    // The Range Explicit Encryption tests require MongoDB server 7.0+ for QE v2.
+    // The tests must not run against a standalone.
+    mongodb: '>=7.0.0',
     topology: '!single'
   }
 };

test/integration/client-side-encryption/client_side_encryption.prose.test.js

@@ -46,7 +46,7 @@ const metadata = {
 const eeMetadata = {
   requires: {
     clientSideEncryption: true,
-    mongodb: '>=6.0.0',
+    mongodb: '>=7.0.0',
     topology: ['replicaset', 'sharded']
   }
 };

test/spec/client-side-encryption/tests/legacy/fle2-BypassQueryAnalysis.json renamed to test/spec/client-side-encryption/tests/legacy/fle2v2-BypassQueryAnalysis.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",
@@ -75,36 +76,6 @@
       "masterKey": {
         "provider": "local"
       }
-    },
-    {
-      "_id": {
-        "$binary": {
-          "base64": "q83vqxI0mHYSNBI0VniQEg==",
-          "subType": "04"
-        }
-      },
-      "keyMaterial": {
-        "$binary": {
-          "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==",
-          "subType": "00"
-        }
-      },
-      "creationDate": {
-        "$date": {
-          "$numberLong": "1648914851981"
-        }
-      },
-      "updateDate": {
-        "$date": {
-          "$numberLong": "1648914851981"
-        }
-      },
-      "status": {
-        "$numberInt": "0"
-      },
-      "masterKey": {
-        "provider": "local"
-      }
     }
   ],
   "tests": [
@@ -133,7 +104,7 @@
               "_id": 1,
               "encryptedIndexed": {
                 "$binary": {
-                  "base64": "BHEBAAAFZAAgAAAAAHb62aV7+mqmaGcotPLdG3KP7S8diFwWMLM/5rYtqLrEBXMAIAAAAAAVJ6OWHRv3OtCozHpt3ZzfBhaxZirLv3B+G8PuaaO4EgVjACAAAAAAsZXWOWA+UiCBbrJNB6bHflB/cn7pWSvwWN2jw4FPeIUFcABQAAAAAMdD1nV2nqeI1eXEQNskDflCy8I7/HvvqDKJ6XxjhrPQWdLqjz+8GosGUsB7A8ee/uG9/guENuL25XD+Fxxkv1LLXtavHOlLF7iW0u9yabqqBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AE0AAAAAq83vqxI0mHYSNBI0VniQEkzZZBBDgeZh+h+gXEmOrSFtVvkUcnHWj/rfPW7iJ0G3UJ8zpuBmUM/VjOMJCY4+eDqdTiPIwX+/vNXegc8FZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsAA==",
+                  "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==",
                   "subType": "06"
                 }
               }
@@ -150,7 +121,7 @@
           "result": [
             {
               "_id": 1,
-              "encryptedIndexed": "value123"
+              "encryptedIndexed": "123"
             }
           ]
         }
@@ -176,7 +147,7 @@
                   "_id": 1,
                   "encryptedIndexed": {
                     "$binary": {
-                      "base64": "BHEBAAAFZAAgAAAAAHb62aV7+mqmaGcotPLdG3KP7S8diFwWMLM/5rYtqLrEBXMAIAAAAAAVJ6OWHRv3OtCozHpt3ZzfBhaxZirLv3B+G8PuaaO4EgVjACAAAAAAsZXWOWA+UiCBbrJNB6bHflB/cn7pWSvwWN2jw4FPeIUFcABQAAAAAMdD1nV2nqeI1eXEQNskDflCy8I7/HvvqDKJ6XxjhrPQWdLqjz+8GosGUsB7A8ee/uG9/guENuL25XD+Fxxkv1LLXtavHOlLF7iW0u9yabqqBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AE0AAAAAq83vqxI0mHYSNBI0VniQEkzZZBBDgeZh+h+gXEmOrSFtVvkUcnHWj/rfPW7iJ0G3UJ8zpuBmUM/VjOMJCY4+eDqdTiPIwX+/vNXegc8FZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsAA==",
+                      "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==",
                       "subType": "06"
                     }
                   }
@@ -230,39 +201,6 @@
             },
             "command_name": "find"
           }
-        },
-        {
-          "command_started_event": {
-            "command": {
-              "find": "datakeys",
-              "filter": {
-                "$or": [
-                  {
-                    "_id": {
-                      "$in": [
-                        {
-                          "$binary": {
-                            "base64": "q83vqxI0mHYSNBI0VniQEg==",
-                            "subType": "04"
-                          }
-                        }
-                      ]
-                    }
-                  },
-                  {
-                    "keyAltNames": {
-                      "$in": []
-                    }
-                  }
-                ]
-              },
-              "$db": "keyvault",
-              "readConcern": {
-                "level": "majority"
-              }
-            },
-            "command_name": "find"
-          }
         }
       ],
       "outcome": {
@@ -276,7 +214,7 @@
               "__safeContent__": [
                 {
                   "$binary": {
-                    "base64": "ThpoKfQ8AkOzkFfNC1+9PF0pY2nIzfXvRdxQgjkNbBw=",
+                    "base64": "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=",
                     "subType": "00"
                   }
                 }

test/spec/client-side-encryption/tests/legacy/fle2-BypassQueryAnalysis.yml renamed to test/spec/client-side-encryption/tests/legacy/fle2v2-BypassQueryAnalysis.yml

@@ -1,12 +1,15 @@
+# Requires libmongocrypt 1.8.0.
 runOn:
-  - minServerVersion: "6.0.0"
+  - minServerVersion: "7.0.0"
+    serverless: "forbid"
+    # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]
 database_name: &database_name "default"
 collection_name: &collection_name "default"
 data: []
 encrypted_fields: &encrypted_fields {'escCollection': 'enxcol_.default.esc', 'eccCollection': 'enxcol_.default.ecc', 'ecocCollection': 'enxcol_.default.ecoc', 'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedIndexed', 'bsonType': 'string', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}, {'keyId': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedUnindexed', 'bsonType': 'string'}]}
-key_vault_data: [{'_id': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}}, {'_id': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}} ]
+key_vault_data: [{'_id': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}} ]
 
 tests:
   - description: "BypassQueryAnalysis decrypts"
@@ -22,16 +25,16 @@ tests:
               "_id": 1,
               "encryptedIndexed": {
                 "$binary": {
-                  # Payload has an IndexKey of key1 and UserKey of key2.
-                  "base64": "BHEBAAAFZAAgAAAAAHb62aV7+mqmaGcotPLdG3KP7S8diFwWMLM/5rYtqLrEBXMAIAAAAAAVJ6OWHRv3OtCozHpt3ZzfBhaxZirLv3B+G8PuaaO4EgVjACAAAAAAsZXWOWA+UiCBbrJNB6bHflB/cn7pWSvwWN2jw4FPeIUFcABQAAAAAMdD1nV2nqeI1eXEQNskDflCy8I7/HvvqDKJ6XxjhrPQWdLqjz+8GosGUsB7A8ee/uG9/guENuL25XD+Fxxkv1LLXtavHOlLF7iW0u9yabqqBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AE0AAAAAq83vqxI0mHYSNBI0VniQEkzZZBBDgeZh+h+gXEmOrSFtVvkUcnHWj/rfPW7iJ0G3UJ8zpuBmUM/VjOMJCY4+eDqdTiPIwX+/vNXegc8FZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsAA==",
+                  # Payload has an IndexKey of key1 and UserKey of key1.
+                  "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==",
                   "subType": "06"
                 }
               }
             }
       - name: find
         arguments:
           filter: { "_id": 1 }
-        result: [{"_id": 1, "encryptedIndexed": "value123" }]
+        result: [{"_id": 1, "encryptedIndexed": "123" }]
     expectations:
       - command_started_event:
           command:
@@ -73,29 +76,7 @@ tests:
             $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
-      - command_started_event:
-          command:
-            find: datakeys
-            filter: {
-                  "$or": [
-                      {
-                          "_id": {
-                              "$in": [
-                                  {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}
-                              ]
-                          }
-                      },
-                      {
-                          "keyAltNames": {
-                              "$in": []
-                          }
-                      }
-                  ]
-              }
-            $db: keyvault
-            readConcern: { level: "majority" }
-          command_name: find
     outcome:
       collection:
         data:
-          - {"_id": 1, "encryptedIndexed": { "$$type": "binData" }, "__safeContent__": [{ "$binary" : { "base64" : "ThpoKfQ8AkOzkFfNC1+9PF0pY2nIzfXvRdxQgjkNbBw=", "subType" : "00" } }] }
+          - {"_id": 1, "encryptedIndexed": { "$$type": "binData" }, "__safeContent__": [{ "$binary" : { "base64" : "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=", "subType" : "00" } }] }