feat(NODE-4950)!: remove bson-ext import and squash commits

Author: nbbeeken

.eslintrc.json

@@ -2,7 +2,7 @@
   "root": true,
   "parser": "@typescript-eslint/parser",
   "parserOptions": {
-    "ecmaVersion": 2019
+    "ecmaVersion": 2020
   },
   "plugins": [
     "simple-import-sort",

.evergreen/config.in.yml

@@ -375,7 +375,7 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          NODE_LTS_NAME=${NODE_LTS_NAME} MSVS_VERSION=${MSVS_VERSION} NPM_OPTIONS=${NPM_OPTIONS}\
+          NODE_LTS_NAME=${NODE_LTS_NAME} NPM_OPTIONS=${NPM_OPTIONS}\
             bash ${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh
     - command: expansions.update
       params:
@@ -434,6 +434,7 @@ functions:
           export DRIVERS_TOOLS="${DRIVERS_TOOLS}"
           export NODE_LTS_NAME='${NODE_LTS_NAME}'
           export MONGODB_URI="${MONGODB_URI}"
+          export TEST_SOCKS5_CSFLE="${TEST_SOCKS5_CSFLE}"
           export SSL="${SSL}"
 
           # Disable xtrace (just in case it was accidentally set).
@@ -451,6 +452,7 @@ functions:
         script: |
           export PROJECT_DIRECTORY="$(pwd)"
           export KRB5_KEYTAB='${gssapi_auth_keytab_base64}'
+          export KRB5_NEW_KEYTAB='${gssapi_auth_new_keytab_base64}'
           export KRB5_PRINCIPAL='${gssapi_auth_principal}'
           export MONGODB_URI='${gssapi_auth_mongodb_uri}'
           export NODE_LTS_NAME='${NODE_LTS_NAME}'

.evergreen/config.yml

@@ -334,7 +334,7 @@ functions:
         working_dir: src
         script: |
           ${PREPARE_SHELL}
-          NODE_LTS_NAME=${NODE_LTS_NAME} MSVS_VERSION=${MSVS_VERSION} NPM_OPTIONS=${NPM_OPTIONS}\
+          NODE_LTS_NAME=${NODE_LTS_NAME} NPM_OPTIONS=${NPM_OPTIONS}\
             bash ${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh
     - command: expansions.update
       params:
@@ -398,6 +398,8 @@ functions:
 
           export MONGODB_URI="${MONGODB_URI}"
 
+          export TEST_SOCKS5_CSFLE="${TEST_SOCKS5_CSFLE}"
+
           export SSL="${SSL}"
 
 
@@ -419,6 +421,7 @@ functions:
         script: |
           export PROJECT_DIRECTORY="$(pwd)"
           export KRB5_KEYTAB='${gssapi_auth_keytab_base64}'
+          export KRB5_NEW_KEYTAB='${gssapi_auth_new_keytab_base64}'
           export KRB5_PRINCIPAL='${gssapi_auth_principal}'
           export MONGODB_URI='${gssapi_auth_mongodb_uri}'
           export NODE_LTS_NAME='${NODE_LTS_NAME}'
@@ -1303,6 +1306,19 @@ tasks:
           TOPOLOGY: replica_set
       - func: bootstrap kms servers
       - func: run socks5 tests
+  - name: test-socks5-csfle
+    tags:
+      - socks5-csfle
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: latest
+          TOPOLOGY: replica_set
+      - func: bootstrap kms servers
+      - func: run socks5 tests
+        vars:
+          TEST_SOCKS5_CSFLE: 'true'
   - name: test-socks5-tls
     tags: []
     commands:
@@ -2132,7 +2148,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: 67bec571c0c21f4db8a96b6bd61cb24dfc87a223
+          CSFLE_GIT_REF: 77b51c00ab4ff58916dd39f55657e1ecc0af281c
   - name: run-custom-csfle-tests-5.0-master
     tags:
       - run-custom-dependency-tests
@@ -2162,7 +2178,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: 67bec571c0c21f4db8a96b6bd61cb24dfc87a223
+          CSFLE_GIT_REF: 77b51c00ab4ff58916dd39f55657e1ecc0af281c
   - name: run-custom-csfle-tests-rapid-master
     tags:
       - run-custom-dependency-tests
@@ -2192,7 +2208,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: 67bec571c0c21f4db8a96b6bd61cb24dfc87a223
+          CSFLE_GIT_REF: 77b51c00ab4ff58916dd39f55657e1ecc0af281c
   - name: run-custom-csfle-tests-latest-master
     tags:
       - run-custom-dependency-tests
@@ -2661,6 +2677,7 @@ buildvariants:
       - test-auth-kerberos
       - test-auth-ldap
       - test-socks5
+      - test-socks5-csfle
       - test-socks5-tls
       - test-zstd-compression
       - test-snappy-compression
@@ -2709,6 +2726,7 @@ buildvariants:
       - test-auth-kerberos
       - test-auth-ldap
       - test-socks5
+      - test-socks5-csfle
       - test-socks5-tls
       - test-tls-support-latest
       - test-tls-support-6.0
@@ -2752,8 +2770,10 @@ buildvariants:
       - test-5.0-load-balanced
       - test-6.0-load-balanced
       - test-latest-load-balanced
+      - test-auth-kerberos
       - test-auth-ldap
       - test-socks5
+      - test-socks5-csfle
       - test-socks5-tls
       - test-tls-support-latest
       - test-tls-support-6.0
@@ -2798,6 +2818,7 @@ buildvariants:
       - test-6.0-load-balanced
       - test-latest-load-balanced
       - test-auth-ldap
+      - test-socks5-csfle
       - test-socks5-tls
       - test-tls-support-latest
       - test-tls-support-6.0
@@ -2809,7 +2830,6 @@ buildvariants:
     run_on: windows-64-vs2019-large
     expansions:
       NODE_LTS_NAME: fermium
-      MSVS_VERSION: 2019
     tasks:
       - test-latest-server
       - test-latest-replica_set
@@ -2851,7 +2871,6 @@ buildvariants:
     run_on: windows-64-vs2019-large
     expansions:
       NODE_LTS_NAME: gallium
-      MSVS_VERSION: 2019
     tasks:
       - test-latest-server
       - test-latest-replica_set
@@ -2891,7 +2910,6 @@ buildvariants:
     run_on: windows-64-vs2019-large
     expansions:
       NODE_LTS_NAME: hydrogen
-      MSVS_VERSION: 2019
     tasks:
       - test-latest-server
       - test-latest-replica_set

.evergreen/generate_evergreen_tasks.js

@@ -28,7 +28,6 @@ const OPERATING_SYSTEMS = [
     name: 'windows-64-vs2019',
     display_name: 'Windows (VS2019)',
     run_on: 'windows-64-vs2019-large',
-    msvsVersion: 2019,
     clientEncryption: false // TODO(NODE-3401): Unskip when Windows no longer fails to launch mongocryptd occasionally
   }
 ].map(osConfig => ({
@@ -39,7 +38,7 @@ const OPERATING_SYSTEMS = [
 }));
 
 // TODO: NODE-3060: enable skipped tests on windows
-const WINDOWS_SKIP_TAGS = new Set(['atlas-connect', 'auth', 'load_balancer']);
+const WINDOWS_SKIP_TAGS = new Set(['atlas-connect', 'auth', 'load_balancer', 'socks5-csfle']);
 
 const TASKS = [];
 const SINGLETON_TASKS = [];
@@ -204,6 +203,27 @@ TASKS.push(
         { func: 'run socks5 tests' }
       ]
     },
+    {
+      name: 'test-socks5-csfle',
+      tags: ['socks5-csfle'],
+      commands: [
+        { func: 'install dependencies' },
+        {
+          func: 'bootstrap mongo-orchestration',
+          vars: {
+            VERSION: 'latest',
+            TOPOLOGY: 'replica_set'
+          }
+        },
+        { func: 'bootstrap kms servers' },
+        {
+          func: 'run socks5 tests',
+          vars: {
+            TEST_SOCKS5_CSFLE: 'true'
+          }
+        }
+      ]
+    },
     {
       name: 'test-socks5-tls',
       tags: [],
@@ -374,7 +394,6 @@ for (const
     run_on,
     nodeVersions = NODE_VERSIONS,
     clientEncryption,
-    msvsVersion
   } of OPERATING_SYSTEMS) {
   const testedNodeVersions = NODE_VERSIONS.filter(version => nodeVersions.includes(version));
   const os = osName.split('-')[0];
@@ -397,9 +416,6 @@ for (const
     if (clientEncryption) {
       expansions.CLIENT_ENCRYPTION = true;
     }
-    if (msvsVersion) {
-      expansions.MSVS_VERSION = msvsVersion;
-    }
 
     BUILD_VARIANTS.push({ name, display_name, run_on, expansions, tasks: taskNames });
   };
@@ -578,8 +594,10 @@ BUILD_VARIANTS.push({
 
 const oneOffFuncAsTasks = []
 
+const FLE_PINNED_COMMIT = '77b51c00ab4ff58916dd39f55657e1ecc0af281c'
+
 for (const version of ['5.0', 'rapid', 'latest']) {
-  for (const ref of ['67bec571c0c21f4db8a96b6bd61cb24dfc87a223', 'master']) {
+  for (const ref of [FLE_PINNED_COMMIT, 'master']) {
     oneOffFuncAsTasks.push({
       name: `run-custom-csfle-tests-${version}-${ref === 'master' ? ref : 'pinned-commit'}`,
       tags: ['run-custom-dependency-tests'],
@@ -667,9 +685,9 @@ for (const variant of BUILD_VARIANTS.filter(
   );
 }
 
-// TODO(NODE-4894): fix kerberos tests on Node18
+// TODO(NODE-5021): Drop support for Kerberos 1.x on in 6.0.0
 for (const variant of BUILD_VARIANTS.filter(
-  variant => variant.expansions && ['hydrogen', 'latest'].includes(variant.expansions.NODE_LTS_NAME)
+  variant => variant.expansions && ['latest'].includes(variant.expansions.NODE_LTS_NAME)
 )) {
   variant.tasks = variant.tasks.filter(
     name => !['test-auth-kerberos'].includes(name)

.evergreen/install-dependencies.sh

@@ -5,19 +5,14 @@ NVM_WINDOWS_URL="https://github.com/coreybutler/nvm-windows/releases/download/1.
 NVM_URL="https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh"
 
 NODE_LTS_NAME=${NODE_LTS_NAME:-fermium}
-MSVS_VERSION=${MSVS_VERSION:-2019}
 NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
-NPM_CACHE_DIR="${NODE_ARTIFACTS_PATH}/npm"
-NPM_TMP_DIR="${NODE_ARTIFACTS_PATH}/tmp"
 
 # this needs to be explicitly exported for the nvm install below
 export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
 export XDG_CONFIG_HOME=${NODE_ARTIFACTS_PATH}
 
 # create node artifacts path if needed
-mkdir -p ${NODE_ARTIFACTS_PATH}
-mkdir -p ${NPM_CACHE_DIR}
-mkdir -p "${NPM_TMP_DIR}"
+mkdir -p "${NODE_ARTIFACTS_PATH}"
 
 function node_lts_to_version() {
   case $1 in
@@ -99,22 +94,12 @@ EOT
   which node || echo "node not found, PATH=$PATH"
   which npm || echo "npm not found, PATH=$PATH"
   npm cache clear --force # Fixes: Cannot read properties of null (reading 'pickAlgorithm') error on windows
-  npm config set msvs_version ${MSVS_VERSION}
-  npm config set scripts-prepend-node-path true
 
 # install Node.js on Linux/MacOS
 else
   curl -o- $NVM_URL | bash
   [ -s "${NVM_DIR}/nvm.sh" ] && source "${NVM_DIR}/nvm.sh"
   nvm install --no-progress "$NODE_VERSION"
-
-  # setup npm cache in a local directory
-  cat <<EOT > .npmrc
-devdir=${NPM_CACHE_DIR}/.node-gyp
-init-module=${NPM_CACHE_DIR}/.npm-init.js
-cache=${NPM_CACHE_DIR}
-tmp=${NPM_TMP_DIR}
-EOT
 fi
 
 npm install ${NPM_OPTIONS}

.evergreen/run-kerberos-tests.sh

@@ -6,16 +6,17 @@ source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
 
 # set up keytab
 mkdir -p "$(pwd)/.evergreen"
-touch "$(pwd)/.evergreen/krb5.conf.empty"
 export KRB5_CONFIG="$(pwd)/.evergreen/krb5.conf.empty"
 echo "Writing keytab"
 # DON'T PRINT KEYTAB TO STDOUT
 set +o verbose
 if [[ "$OSTYPE" == "darwin"* ]]; then
-    echo ${KRB5_KEYTAB} | base64 -D > "$(pwd)/.evergreen/drivers.keytab"
+    echo ${KRB5_NEW_KEYTAB} | base64 -D > "$(pwd)/.evergreen/drivers.keytab"
 else
-    echo ${KRB5_KEYTAB} | base64 -d > "$(pwd)/.evergreen/drivers.keytab"
+    echo ${KRB5_NEW_KEYTAB} | base64 -d > "$(pwd)/.evergreen/drivers.keytab"
 fi
+echo "Running kdestroy"
+kdestroy -A
 echo "Running kinit"
 kinit -k -t "$(pwd)/.evergreen/drivers.keytab" -p ${KRB5_PRINCIPAL}
 

.evergreen/run-kms-servers.sh

@@ -1,7 +1,7 @@
 cd ${DRIVERS_TOOLS}/.evergreen/csfle
 . ./activate-kmstlsvenv.sh
 # by default it always runs on port 5698
-./kmstlsvenv/bin/python3 -u kms_kmip_server.py &
-./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000  &
-./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001  &
-./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
+python -u kms_kmip_server.py &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000  &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001  &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &

.evergreen/run-socks5-tests.sh

@@ -18,9 +18,9 @@ fi
 
 # Grab a connection string that only refers to *one* of the hosts in MONGODB_URI
 FIRST_HOST=$(node -p 'new (require("mongodb-connection-string-url").default)(process.env.MONGODB_URI).hosts[0]')
-# Use localhost:12345 as the URL for the single host that we connect to,
+# Use 127.0.0.1:12345 as the URL for the single host that we connect to,
 # we configure the Socks5 proxy server script to redirect from this to FIRST_HOST
-export MONGODB_URI_SINGLEHOST="mongodb://localhost:12345/"
+export MONGODB_URI_SINGLEHOST="mongodb://127.0.0.1:12345/"
 
 # Compute path to socks5 fake server script in a way that works on Windows
 SOCKS5_SERVER_SCRIPT="$DRIVERS_TOOLS/.evergreen/socks5srv.py"
@@ -29,22 +29,28 @@ if [ "Windows_NT" = "$OS" ]; then
 fi
 
 # First, test with Socks5 + authentication required
-"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1080 --auth username:p4ssw0rd --map "localhost:12345 to $FIRST_HOST" &
-PID=$!
-env SOCKS5_CONFIG='["localhost",1080,"username","p4ssw0rd"]' npm run check:socks5
-[ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
-  env MONGODB_URI='mongodb://localhost:12345/?proxyHost=localhost&proxyUsername=username&proxyPassword=p4ssw0rd' \
+"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1080 --auth username:p4ssw0rd --map "127.0.0.1:12345 to $FIRST_HOST" &
+SOCKS5_PROXY_PID=$!
+if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then
+  [ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
+  env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyUsername=username&proxyPassword=p4ssw0rd' \
   bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
-kill $PID
+else
+  env SOCKS5_CONFIG='["127.0.0.1",1080,"username","p4ssw0rd"]' npm run check:socks5
+fi
+kill $SOCKS5_PROXY_PID
 
 # Second, test with Socks5 + no authentication
-"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1081 --map "localhost:12345 to $FIRST_HOST" &
-PID=$!
-env SOCKS5_CONFIG='["localhost",1081]' npm run check:socks5
-[ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
-  env MONGODB_URI='mongodb://localhost:12345/?proxyHost=localhost&proxyPort=1081' \
-  bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
-kill $PID
+"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1081 --map "127.0.0.1:12345 to $FIRST_HOST" &
+SOCKS5_PROXY_PID=$!
+if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then
+  [ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
+    env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyPort=1081' \
+    bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
+else
+  env SOCKS5_CONFIG='["127.0.0.1",1081]' npm run check:socks5
+fi
+kill $SOCKS5_PROXY_PID
 
 # TODO: It might be worth using something more robust to control
 # the Socks5 proxy server script's lifetime

.evergreen/run-tests.sh

@@ -52,7 +52,7 @@ else
   source "$DRIVERS_TOOLS"/.evergreen/csfle/set-temp-creds.sh
 fi
 
-npm install mongodb-client-encryption@"2.4.0-alpha.2"
+npm install mongodb-client-encryption@"2.5.0"
 npm install @mongodb-js/zstd
 npm install snappy
 

.gitignore

@@ -87,3 +87,4 @@ etc/docs/build
 !docs/**/*.png
 !docs/**/*.css
 !docs/**/*.js
+.nvmrc