From 959dd744f6c0c729040f7654b5319d7490854d49 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 28 May 2025 16:15:24 +0100 Subject: [PATCH 01/12] Remove git-archive.sh and fix archiving logic JAVA-5882 --- .evergreen/.evg.yml | 85 +++++++++++++------ .evergreen/git-archive.sh | 20 ----- .../kotlin/conventions/git-version.gradle.kts | 6 +- 3 files changed, 66 insertions(+), 45 deletions(-) delete mode 100755 .evergreen/git-archive.sh diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index c20533906e..66e5b63c2e 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -61,6 +61,8 @@ functions: export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" export UPLOAD_BUCKET="${project}" export PROJECT_DIRECTORY="$(pwd)" + export ARCHIVE_FILE_NAME="mongo-java-driver.tgz" + export ARCHIVE_FILE_PATH="/tmp/$ARCHIVE_FILE_NAME" cat < expansion.yml CURRENT_VERSION: "$CURRENT_VERSION" @@ -69,6 +71,8 @@ functions: MONGODB_BINARIES: "$MONGODB_BINARIES" UPLOAD_BUCKET: "$UPLOAD_BUCKET" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" + ARCHIVE_FILE_NAME: "$ARCHIVE_FILE_NAME" + ARCHIVE_FILE_PATH: "$ARCHIVE_FILE_PATH" PREPARE_SHELL: | set -o errexit set -o xtrace @@ -77,10 +81,11 @@ functions: export MONGODB_BINARIES="$MONGODB_BINARIES" export UPLOAD_BUCKET="$UPLOAD_BUCKET" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" export PROJECT="${project}" + export ARCHIVE_FILE_NAME="$ARCHIVE_FILE_NAME" + export ARCHIVE_FILE_PATH="$ARCHIVE_FILE_PATH" EOT # See what we've done cat expansion.yml @@ -266,7 +271,7 @@ functions: . ./activate-authawsvenv.sh python ./lib/aws_assign_instance_profile.py fi - + "run tests": - command: shell.exec type: test @@ -766,6 +771,15 @@ functions: bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh || true rm -rf $DRIVERS_TOOLS || true + "create archive tar file": + - command: shell.exec + params: + working_dir: "src" + script: | + echo "Creating archive tar file at ${ARCHIVE_FILE_PATH}" + tar --exclude="**build/" --exclude-vcs -czf "${ARCHIVE_FILE_PATH}" . + echo "Created archive tar file at ${ARCHIVE_FILE_PATH}" + "fix absolute paths": - command: shell.exec params: @@ -830,12 +844,13 @@ functions: type: test params: shell: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + working_dir: "src" + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "ARCHIVE_FILE_PATH"] script: |- set -o errexit ${PREPARE_SHELL} export K8S_VARIANT=${VARIANT} - export K8S_DRIVERS_TAR_FILE=$(./.evergreen/git-archive.sh) + export K8S_DRIVERS_TAR_FILE=$ARCHIVE_FILE_PATH export K8S_TEST_CMD="OIDC_ENV=k8s VARIANT=${VARIANT} ./.evergreen/run-mongodb-oidc-test.sh" bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh @@ -929,29 +944,35 @@ tasks: commands: - command: shell.exec params: - shell: bash + shell: "bash" + working_dir: "src" + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "ARCHIVE_FILE_PATH"] env: JAVA_HOME: ${JAVA_HOME} script: |- set -o errexit ${PREPARE_SHELL} - export AZUREOIDC_DRIVERS_TAR_FILE=$(./.evergreen/git-archive.sh) + export AZUREOIDC_DRIVERS_TAR_FILE=$ARCHIVE_FILE_PATH export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" + tar --exclude="**build/" --exclude-vcs -czf $AZUREOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh - name: "oidc-auth-test-gcp" commands: - command: shell.exec params: - shell: bash + shell: "bash" + working_dir: "src" + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "ARCHIVE_FILE_PATH"] script: |- set -o errexit ${PREPARE_SHELL} - export GCPOIDC_DRIVERS_TAR_FILE=$(./.evergreen/git-archive.sh) + export GCPOIDC_DRIVERS_TAR_FILE=$ARCHIVE_FILE_PATH # Define the command to run on the VM. # Ensure that we source the environment file created for us, set up any other variables we need, # and then run our test suite on the vm. export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" + tar --exclude="**build/" --exclude-vcs -czf $GCPOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh - name: "oidc-auth-test-k8s" @@ -971,8 +992,6 @@ tasks: - func: "oidc-auth-test-k8s-func" vars: VARIANT: gke - params: - include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] - name: serverless-test commands: @@ -1674,11 +1693,10 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - tar czf /tmp/mongo-java-driver.tgz . - GCPKMS_SRC=/tmp/mongo-java-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh + GCPKMS_SRC=$ARCHIVE_FILE_PATH GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" - GCPKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="tar xf $ARCHIVE_FILE_NAME" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" - command: shell.exec type: test @@ -1706,11 +1724,11 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - tar czf /tmp/mongo-csharp-driver.tgz . - AZUREKMS_SRC=/tmp/mongo-csharp-driver.tgz AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh + tar --exclude="**build/" --exclude-vcs -czf $ARCHIVE_FILE_PATH . + AZUREKMS_SRC=$ARCHIVE_FILE_PATH AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" - AZUREKMS_CMD="tar xf mongo-csharp-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + AZUREKMS_CMD="tar xf $ARCHIVE_FILE_NAME" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" - command: shell.exec type: test @@ -1971,6 +1989,7 @@ task_groups: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file - command: shell.exec params: shell: "bash" @@ -2007,6 +2026,7 @@ task_groups: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file - command: shell.exec params: shell: "bash" @@ -2046,6 +2066,10 @@ task_groups: - testazurekms-task - name: test-oidc-task-group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + teardown_task_can_fail_task: true + teardown_task_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources @@ -2065,19 +2089,26 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/teardown.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 tasks: - oidc-auth-test - name: test-oidc-azure-task-group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + teardown_task_can_fail_task: true + teardown_task_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} - command: subprocess.exec params: binary: bash + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] env: AZUREOIDC_VMNAME_PREFIX: "JAVA_DRIVER" args: @@ -2088,19 +2119,26 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/delete-vm.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 tasks: - oidc-auth-test-azure - name: test-oidc-gcp-task-group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + teardown_task_can_fail_task: true + teardown_task_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} - command: subprocess.exec params: binary: bash + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] env: GCPOIDC_VMNAME_PREFIX: "JAVA_DRIVER" GCPKMS_MACHINETYPE: "e2-medium" # comparable elapsed time to Azure; default was starved, caused timeouts @@ -2112,8 +2150,6 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 tasks: - oidc-auth-test-gcp @@ -2121,11 +2157,12 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 teardown_task_can_fail_task: true - teardown_group_timeout_secs: 180 + teardown_task_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file - command: ec2.assume_role params: role_arn: ${aws_test_secrets_role} @@ -2135,7 +2172,7 @@ task_groups: include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh - teardown_group: + teardown_task: - command: subprocess.exec params: binary: bash diff --git a/.evergreen/git-archive.sh b/.evergreen/git-archive.sh deleted file mode 100755 index 5c22c9170a..0000000000 --- a/.evergreen/git-archive.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Exit the script with error if any of the commands fail -set -o errexit - -# Returns the path to the root archive file which includes all git submodules. - -echo "Creating root archive" -export GIT_ARCHIVE_FILE="/tmp/mongo-java-driver.tar" - -# create root archive -git archive --output $GIT_ARCHIVE_FILE HEAD - -echo "Appending submodule archives" -git submodule status --recursive | awk '{ print $2 }' | xargs tar -rf $GIT_ARCHIVE_FILE - -echo "Appending .git directory to the root archive" -tar -rf $GIT_ARCHIVE_FILE .git - -echo "$GIT_ARCHIVE_FILE" diff --git a/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts b/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts index 5370f59cea..9ddfd25cab 100644 --- a/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts +++ b/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts @@ -19,10 +19,14 @@ package conventions val gitVersion: Provider = providers - .exec { commandLine("git", "describe", "--tags", "--always", "--dirty") } + .exec { + commandLine("git", "describe", "--tags", "--always", "--dirty") + isIgnoreExitValue = true + } .standardOutput .asText .map { it.trim().removePrefix("r") } + .orElse("UNKNOWN") // Allows access to gitVersion extension to other conventions extensions.add("gitVersion", gitVersion) From a66b24204e9ab957497257c192cac7d9956dda83 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 May 2025 15:25:40 +0100 Subject: [PATCH 02/12] Evergreen reorganization Normalize naming Group functions JAVA-5889 --- .evergreen/.evg.yml | 3473 ++++++++++++++++--------------- .evergreen/run-kms-tls-tests.sh | 1 + 2 files changed, 1766 insertions(+), 1708 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 66e5b63c2e..f732d1efd5 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -9,8 +9,8 @@ stepback: true # Mark a failure as a system/bootstrap failure (purple box) rather then a task # failure by default. -# Actual testing tasks are marked with `type: test` -command_type: system +# Actual testing tasks are marked with `type: "test"` +command_type: "system" # Protect ourselves against rogue test case, or curl gone wild, that runs forever exec_timeout_secs: 3600 @@ -23,7 +23,12 @@ timeout: ls -la functions: - "fetch source": + + # + # Start up and teardown functions + # + + "fetch-source": # Executes git clone and applies the submitted patch, if any - command: git.get_project params: @@ -41,64 +46,64 @@ functions: - command: shell.exec params: working_dir: "src" - shell: bash + shell: "bash" script: | - # Get the current unique version of this checkout - if [ "${is_patch}" = "true" ]; then - CURRENT_VERSION=$(git describe)-patch-${version_id} - else - CURRENT_VERSION=latest - fi - - export DRIVERS_TOOLS="$(pwd)/../drivers-tools" - - # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory - if [ "Windows_NT" == "$OS" ]; then # Magic variable in cygwin - export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) - fi - - export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" - export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" - export UPLOAD_BUCKET="${project}" - export PROJECT_DIRECTORY="$(pwd)" - export ARCHIVE_FILE_NAME="mongo-java-driver.tgz" - export ARCHIVE_FILE_PATH="/tmp/$ARCHIVE_FILE_NAME" - - cat < expansion.yml - CURRENT_VERSION: "$CURRENT_VERSION" - DRIVERS_TOOLS: "$DRIVERS_TOOLS" - MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" - MONGODB_BINARIES: "$MONGODB_BINARIES" - UPLOAD_BUCKET: "$UPLOAD_BUCKET" - PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" - ARCHIVE_FILE_NAME: "$ARCHIVE_FILE_NAME" - ARCHIVE_FILE_PATH: "$ARCHIVE_FILE_PATH" - PREPARE_SHELL: | - set -o errexit - set -o xtrace - export DRIVERS_TOOLS="$DRIVERS_TOOLS" - export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" - export MONGODB_BINARIES="$MONGODB_BINARIES" - export UPLOAD_BUCKET="$UPLOAD_BUCKET" - export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" - export PATH="$MONGODB_BINARIES:$PATH" - export PROJECT="${project}" - export ARCHIVE_FILE_NAME="$ARCHIVE_FILE_NAME" - export ARCHIVE_FILE_PATH="$ARCHIVE_FILE_PATH" - EOT - # See what we've done - cat expansion.yml + # Get the current unique version of this checkout + if [ "${is_patch}" = "true" ]; then + CURRENT_VERSION=$(git describe)-patch-${version_id} + else + CURRENT_VERSION=latest + fi + + export DRIVERS_TOOLS="$(pwd)/../drivers-tools" + + # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory + if [ "Windows_NT" == "$OS" ]; then # Magic variable in cygwin + export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) + fi + + export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" + export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" + export UPLOAD_BUCKET="${project}" + export PROJECT_DIRECTORY="$(pwd)" + export ARCHIVE_FILE_NAME="mongo-java-driver.tgz" + export ARCHIVE_FILE_PATH="/tmp/$ARCHIVE_FILE_NAME" + + cat < expansion.yml + CURRENT_VERSION: "$CURRENT_VERSION" + DRIVERS_TOOLS: "$DRIVERS_TOOLS" + MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" + MONGODB_BINARIES: "$MONGODB_BINARIES" + UPLOAD_BUCKET: "$UPLOAD_BUCKET" + PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" + ARCHIVE_FILE_NAME: "$ARCHIVE_FILE_NAME" + ARCHIVE_FILE_PATH: "$ARCHIVE_FILE_PATH" + PREPARE_SHELL: | + set -o errexit + set -o xtrace + export DRIVERS_TOOLS="$DRIVERS_TOOLS" + export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" + export MONGODB_BINARIES="$MONGODB_BINARIES" + export UPLOAD_BUCKET="$UPLOAD_BUCKET" + export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" + export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" + export PATH="$MONGODB_BINARIES:$PATH" + export PROJECT="${project}" + export ARCHIVE_FILE_NAME="$ARCHIVE_FILE_NAME" + export ARCHIVE_FILE_PATH="$ARCHIVE_FILE_PATH" + EOT + # See what we've done + cat expansion.yml # Load the expansion file to make an evergreen variable with the current unique version - command: expansions.update params: file: src/expansion.yml - "prepare resources": + "prepare-resources": - command: shell.exec params: - shell: bash + shell: "bash" script: | ${PREPARE_SHELL} rm -rf $DRIVERS_TOOLS @@ -110,23 +115,196 @@ functions: fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config - "exec script" : + "fix-absolute-paths": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do + perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename + done + + "create-archive-tar-file": - command: shell.exec - type: test params: working_dir: "src" - shell: bash + script: | + echo "Creating archive tar file at ${ARCHIVE_FILE_PATH}" + tar --exclude="**build/" --exclude-vcs -czf "${ARCHIVE_FILE_PATH}" . + echo "Created archive tar file at ${ARCHIVE_FILE_PATH}" + + "start-mongo-orchestration": + - command: shell.exec + params: script: | ${PREPARE_SHELL} - ${PROJECT_DIRECTORY}/${file} + REQUIRE_API_VERSION=${REQUIRE_API_VERSION} LOAD_BALANCER=${LOAD_BALANCER} MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} \ + AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ + bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + # run-orchestration generates expansion file with the MONGODB_URI for the cluster + - command: expansions.update + params: + file: mo-expansion.yml + "stop-mongo-orchestration": + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh || true + + "start-mongohoused": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: shell.exec + params: + include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] + script: | + DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh + - command: shell.exec + params: + script: | + DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh - "upload mo artifacts": + "stop-mongohoused": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: shell.exec + params: + include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] + script: | + DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/teardown.sh || true + + + "start-serverless": + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh ${VAULT_NAME} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: serverless-expansion.yml + "stop-serverless": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh || true + + "start-load-balancer": + - command: shell.exec + params: + script: | + DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start + - command: expansions.update + params: + file: lb-expansion.yml + "stop-load-balancer": + - command: shell.exec + params: + script: | + cd ${DRIVERS_TOOLS}/.evergreen + DRIVERS_TOOLS=${DRIVERS_TOOLS} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop || true + + "start-kms-mock-server": + - command: shell.exec + params: + background: true + shell: "bash" + script: | + ${PREPARE_SHELL} + cd ${DRIVERS_TOOLS}/.evergreen/csfle + . ./activate-kmstlsvenv.sh + python -u kms_http_server.py -v --ca_file ../x509gen/ca.pem --cert_file ../x509gen/${CERT_FILE} --port 8000 + + "start-kms-kmip-server": + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + cd ${DRIVERS_TOOLS}/.evergreen/csfle + . ./activate-kmstlsvenv.sh + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/csfle + . ./activate-kmstlsvenv.sh + python -u kms_kmip_server.py + + "stop-aws": + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" + if [ -f "./aws_e2e_setup.json" ]; then + . ./activate-authawsvenv.sh + python ./lib/aws_assign_instance_profile.py + fi + + "cleanup": + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + rm -rf $DRIVERS_TOOLS || true + + "add-aws-auth-variables-to-file": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: shell.exec + type: "test" + params: + include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + cd $DRIVERS_TOOLS/.evergreen/auth_aws + ./setup_secrets.sh drivers/aws_auth + + "start-csfle-servers": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: subprocess.exec + params: + working_dir: "src" + binary: "bash" + include_expansions_in_env: ["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN"] + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh + + "stop-csfle-servers": + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh || true + + # + # Publishing / uploading functions + # + + "upload-mo-artifacts": - command: ec2.assume_role params: role_arn: ${UPLOAD_MO_ARTIFACTS_ROLE_ARN} - command: shell.exec params: - shell: bash + shell: "bash" script: | ${PREPARE_SHELL} find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf mongodb-logs.tar.gz @@ -139,7 +317,7 @@ functions: remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz bucket: mciuploads permissions: public-read - content_type: ${content_type|application/x-gzip} + content_type: "${content_type|application/x-gzip}" display_name: "mongodb-logs.tar.gz" - command: s3.put params: @@ -150,10 +328,10 @@ functions: remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log bucket: mciuploads permissions: public-read - content_type: ${content_type|text/plain} + content_type: "${content_type|text/plain}" display_name: "orchestration.log" - "create and upload SSDLC release assets": + "create-and-upload-SSDLC-release-assets": - command: shell.exec params: shell: "bash" @@ -176,8 +354,8 @@ functions: bucket: java-driver-release-assets region: us-west-1 permissions: private - content_type: text/markdown - display_name: ssdlc_compliance_report.md + content_type: "text/markdown" + display_name: "ssdlc_compliance_report.md" - command: s3.put params: aws_key: ${AWS_ACCESS_KEY_ID} @@ -190,91 +368,109 @@ functions: bucket: java-driver-release-assets region: us-west-1 permissions: private - content_type: application/sarif+json + content_type: "application/sarif+json" display_name: - "upload test results": + "upload-test-results": - command: attach.xunit_results params: file: ./src/*/build/test-results/*/TEST-*.xml - "bootstrap mongo-orchestration": + "trace-artifacts": - command: shell.exec params: + working_dir: "src" script: | - ${PREPARE_SHELL} - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} LOAD_BALANCER=${LOAD_BALANCER} MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ - bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh - # run-orchestration generates expansion file with the MONGODB_URI for the cluster + PRODUCT_VERSION="$(echo -n "$(git describe --tags --always --dirty)" | cut -c 2-)" + cat > ssdlc-expansions.yml < ssdlc-expansions.yml < Date: Thu, 5 Jun 2025 15:09:02 +0100 Subject: [PATCH 03/12] Removed unused task Leftover from JAVA-4502 --- .evergreen/.evg.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index f732d1efd5..b7f8054d75 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -1127,16 +1127,6 @@ tasks: - func: "add-aws-auth-variables-to-file" - func: "run-aws-auth-test-with-web-identity-credentials" - - name: "aws-ECS-auth-test-task" - commands: - - func: "start-mongo-orchestration" - vars: - AUTH: "auth" - ORCHESTRATION_FILE: "auth-aws.json" - TOPOLOGY: "server" - - func: "add-aws-auth-variables-to-file" - - func: "run-aws-ECS-auth-test" - - name: "test-ocsp-rsa-valid-cert-server-staples-task" tags: [ "ocsp" ] commands: From 67784cd93797731375ff9fea5e0a9dbcb069ef54 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 2 Jun 2025 15:44:01 +0100 Subject: [PATCH 04/12] Update the evergreen build to use the AWS secrets manager JAVA-5886 --- .evergreen/.evg.yml | 134 +++++------------- .evergreen/javaConfig.bash | 19 --- .evergreen/publish.sh | 2 +- .evergreen/run-atlas-data-lake-test.sh | 2 +- ...run-atlas-search-index-management-tests.sh | 2 +- .evergreen/run-atlas-search-tests.sh | 2 +- .evergreen/run-connectivity-tests.sh | 2 +- .evergreen/run-csfle-aws-from-environment.sh | 3 +- .../run-csfle-tests-with-mongocryptd.sh | 2 +- .evergreen/run-deployed-lambda-aws-tests.sh | 2 +- .evergreen/run-graalvm-native-image-app.sh | 2 +- .evergreen/run-gssapi-auth-test.sh | 2 +- .evergreen/run-kms-tls-tests.sh | 2 +- .evergreen/run-kotlin-tests.sh | 2 +- .evergreen/run-load-balancer-tests.sh | 2 +- .evergreen/run-mongodb-aws-ecs-test.sh | 2 +- .evergreen/run-mongodb-aws-test.sh | 2 +- .evergreen/run-mongodb-oidc-test.sh | 2 +- .evergreen/run-ocsp-test.sh | 2 +- .evergreen/run-perf-tests.sh | 2 +- .evergreen/run-plain-auth-test.sh | 2 +- .evergreen/run-reactive-streams-tck-tests.sh | 2 +- .evergreen/run-scala-tests.sh | 2 +- .evergreen/run-serverless-tests.sh | 2 +- .evergreen/run-socket-tests.sh | 2 +- .evergreen/run-socks5-tests.sh | 4 +- .evergreen/run-tests.sh | 2 +- .evergreen/setup-env.bash | 51 +++++++ .evergreen/ssdlc-report.sh | 2 +- .evergreen/static-checks.sh | 2 +- ...bstractClientSideEncryptionKmsTlsTest.java | 4 +- 31 files changed, 118 insertions(+), 147 deletions(-) delete mode 100644 .evergreen/javaConfig.bash create mode 100644 .evergreen/setup-env.bash diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index b7f8054d75..4032ed34b4 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -124,6 +124,11 @@ functions: perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename done + "assume-aws-test-secrets-role": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + "create-archive-tar-file": - command: shell.exec params: @@ -154,9 +159,6 @@ functions: bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh || true "start-mongohoused": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - command: shell.exec params: include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] @@ -168,9 +170,6 @@ functions: DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh "stop-mongohoused": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - command: shell.exec params: include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] @@ -182,6 +181,7 @@ functions: - command: shell.exec params: shell: "bash" + include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] script: | ${PREPARE_SHELL} bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh ${VAULT_NAME} @@ -192,6 +192,8 @@ functions: "stop-serverless": - command: shell.exec params: + shell: "bash" + include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] script: | ${PREPARE_SHELL} bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh || true @@ -211,34 +213,6 @@ functions: cd ${DRIVERS_TOOLS}/.evergreen DRIVERS_TOOLS=${DRIVERS_TOOLS} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop || true - "start-kms-mock-server": - - command: shell.exec - params: - background: true - shell: "bash" - script: | - ${PREPARE_SHELL} - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - python -u kms_http_server.py -v --ca_file ../x509gen/ca.pem --cert_file ../x509gen/${CERT_FILE} --port 8000 - - "start-kms-kmip-server": - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - - command: shell.exec - params: - shell: "bash" - background: true - script: | - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - python -u kms_kmip_server.py - "stop-aws": - command: shell.exec params: @@ -260,9 +234,6 @@ functions: rm -rf $DRIVERS_TOOLS || true "add-aws-auth-variables-to-file": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - command: shell.exec type: "test" params: @@ -463,8 +434,6 @@ functions: echo "Response Body: $response_body" echo "HTTP Status: $http_status" - - # # Test functions # @@ -474,25 +443,11 @@ functions: params: working_dir: "src" env: - AWS_ACCESS_KEY_ID: ${aws_access_key_id} - AWS_SECRET_ACCESS_KEY: ${aws_secret_access_key} - AWS_ACCESS_KEY_ID_AWS_KMS_NAMED: ${aws_access_key_id_2} - AWS_SECRET_ACCESS_KEY_AWS_KMS_NAMED: ${aws_secret_access_key_2} - AWS_DEFAULT_REGION: us-east-1 - AZURE_TENANT_ID: ${azure_tenant_id} - AZURE_CLIENT_ID: ${azure_client_id} - AZURE_CLIENT_SECRET: ${azure_client_secret} - GCP_EMAIL: ${gcp_email} - GCP_PRIVATE_KEY: ${gcp_private_key} AZUREKMS_KEY_VAULT_ENDPOINT: ${testazurekms_keyvaultendpoint} AZUREKMS_KEY_NAME: ${testazurekms_keyname} script: | ${PREPARE_SHELL} - . ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh - export AWS_TEMP_ACCESS_KEY_ID=$CSFLE_AWS_TEMP_ACCESS_KEY_ID - export AWS_TEMP_SECRET_ACCESS_KEY=$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY - export AWS_TEMP_SESSION_TOKEN=$CSFLE_AWS_TEMP_SESSION_TOKEN export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} AUTH="${AUTH}" SSL="${SSL}" MONGODB_URI="${MONGODB_URI}" TOPOLOGY="${TOPOLOGY}" \ @@ -816,9 +771,6 @@ functions: type: "test" params: working_dir: "src" - env: - AWS_ACCESS_KEY_ID: ${aws_access_key_id} - AWS_SECRET_ACCESS_KEY: ${aws_secret_access_key} script: | ${PREPARE_SHELL} set +o xtrace @@ -830,26 +782,10 @@ functions: params: working_dir: "src" env: - AWS_ACCESS_KEY_ID: ${aws_access_key_id} - AWS_SECRET_ACCESS_KEY: ${aws_secret_access_key} - AWS_ACCESS_KEY_ID_AWS_KMS_NAMED: ${aws_access_key_id_2} - AWS_SECRET_ACCESS_KEY_AWS_KMS_NAMED: ${aws_secret_access_key_2} - AWS_DEFAULT_REGION: us-east-1 - AZURE_TENANT_ID: ${azure_tenant_id} - AZURE_CLIENT_ID: ${azure_client_id} - AZURE_CLIENT_SECRET: ${azure_client_secret} - GCP_EMAIL: ${gcp_email} - GCP_PRIVATE_KEY: ${gcp_private_key} AZUREKMS_KEY_VAULT_ENDPOINT: ${testazurekms_keyvaultendpoint} AZUREKMS_KEY_NAME: ${testazurekms_keyname} script: | ${PREPARE_SHELL} - . ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh - - export AWS_TEMP_ACCESS_KEY_ID=$CSFLE_AWS_TEMP_ACCESS_KEY_ID - export AWS_TEMP_SECRET_ACCESS_KEY=$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY - export AWS_TEMP_SESSION_TOKEN=$CSFLE_AWS_TEMP_SESSION_TOKEN - MONGODB_URI="${MONGODB_URI}" JAVA_VERSION="${JAVA_VERSION}" .evergreen/run-csfle-tests-with-mongocryptd.sh "run-perf-tests": @@ -899,13 +835,15 @@ pre: - func: "fix-absolute-paths" post: + - func: "stop-mongo-orchestration" - func: "upload-mo-artifacts" - func: "upload-test-results" + - func: "assume-aws-test-secrets-role" - func: "stop-load-balancer" - func: "stop-serverless" - func: "stop-aws" - - func: "stop-mongo-orchestration" - func: "stop-mongohoused" + - func: "stop-csfle-servers" - func: "cleanup" tasks: @@ -936,7 +874,7 @@ tasks: - name: "test-legacy-task" commands: - - func: "start-kms-kmip-server" + - func: "start-csfle-servers" - func: "start-mongo-orchestration" - func: "run-tests" vars: @@ -944,7 +882,7 @@ tasks: - name: "test-sync-task" commands: - - func: "start-kms-kmip-server" + - func: "start-csfle-servers" - func: "start-mongo-orchestration" - func: "run-tests" vars: @@ -952,7 +890,7 @@ tasks: - name: "test-reactive-task" commands: - - func: "start-kms-kmip-server" + - func: "start-csfle-servers" - func: "start-mongo-orchestration" - func: "run-tests" vars: @@ -1036,10 +974,7 @@ tasks: # Might exceed 1 hour of execution. exec_timeout_secs: 7200 commands: - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - duration_seconds: 1800 + - func: "assume-aws-test-secrets-role" - func: "run-oidc-auth-test-k8s-test" vars: VARIANT: eks @@ -1052,12 +987,13 @@ tasks: - name: "serverless-test-task" commands: + - func: "assume-aws-test-secrets-role" - func: "start-serverless" - func: "run-serverless-tests" - name: "accept-api-version-2-test-task" commands: - - func: "start-kms-kmip-server" + - func: "start-csfle-servers" - func: "start-mongo-orchestration" vars: ORCHESTRATION_FILE: "versioned-api-testing.json" @@ -1074,6 +1010,7 @@ tasks: AUTH: "auth" ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" + - func: "assume-aws-test-secrets-role" - func: "add-aws-auth-variables-to-file" - func: "run-aws-auth-test-with-regular-aws-credentials" @@ -1084,6 +1021,7 @@ tasks: AUTH: "auth" ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" + - func: "assume-aws-test-secrets-role" - func: "add-aws-auth-variables-to-file" - func: "run-aws-auth-test-with-assume-role-credentials" @@ -1094,6 +1032,7 @@ tasks: AUTH: "auth" ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" + - func: "assume-aws-test-secrets-role" - func: "add-aws-auth-variables-to-file" - func: "run-aws-auth-test-with-aws-credentials-as-environment-variables" @@ -1104,6 +1043,7 @@ tasks: AUTH: "auth" ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" + - func: "assume-aws-test-secrets-role" - func: "add-aws-auth-variables-to-file" - func: "run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables" @@ -1114,6 +1054,7 @@ tasks: AUTH: "auth" ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" + - func: "assume-aws-test-secrets-role" - func: "add-aws-auth-variables-to-file" - func: "run-aws-auth-test-with-aws-EC2 credentials" @@ -1124,6 +1065,7 @@ tasks: AUTH: "auth" ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" + - func: "assume-aws-test-secrets-role" - func: "add-aws-auth-variables-to-file" - func: "run-aws-auth-test-with-web-identity-credentials" @@ -1526,6 +1468,7 @@ tasks: - name: "atlas-data-lake-task" commands: + - func: "assume-aws-test-secrets-role" - func: "start-mongohoused" - command: shell.exec type: "test" @@ -1664,9 +1607,7 @@ tasks: TOPOLOGY: "server" AUTH: "noauth" SSL: "nossl" - - func: "start-kms-mock-server" - vars: - CERT_FILE: "expired.pem" + - func: "start-csfle-servers" - func: "run-kms-tls-test" vars: KMS_TLS_ERROR_TYPE: "expired" @@ -1682,7 +1623,7 @@ tasks: TOPOLOGY: "server" AUTH: "noauth" SSL: "nossl" - - func: "start-kms-mock-server" + - func: "start-csfle-servers" vars: CERT_FILE: "wrong-host.pem" - func: "run-kms-tls-test" @@ -1695,6 +1636,7 @@ tasks: - name: "test-csfle-aws-from-environment-task" tags: [ "csfle-aws-from-environment" ] commands: + - func: "start-csfle-servers" - func: "start-mongo-orchestration" vars: TOPOLOGY: "server" @@ -1704,7 +1646,7 @@ tasks: - name: "csfle-tests-with-mongocryptd-task" commands: - - func: "start-kms-kmip-server" + - func: "start-csfle-servers" - func: "start-mongo-orchestration" - func: "run-csfle-tests-with-mongocryptd" @@ -2054,7 +1996,7 @@ task_groups: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh || true tasks: - "test-gcp-kms-task" @@ -2066,9 +2008,11 @@ task_groups: - func: "prepare-resources" - func: "fix-absolute-paths" - func: "create-archive-tar-file" + - func: "assume-aws-test-secrets-role" - command: shell.exec params: shell: "bash" + include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] script: | ${PREPARE_SHELL} echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey @@ -2113,9 +2057,7 @@ task_groups: - func: "fetch-source" - func: "prepare-resources" - func: "fix-absolute-paths" - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} + - func: "assume-aws-test-secrets-role" - command: subprocess.exec params: binary: bash @@ -2141,9 +2083,7 @@ task_groups: - func: "prepare-resources" - func: "fix-absolute-paths" - func: "create-archive-tar-file" - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} + - func: "assume-aws-test-secrets-role" - command: subprocess.exec params: binary: bash @@ -2171,9 +2111,7 @@ task_groups: - func: "prepare-resources" - func: "fix-absolute-paths" - func: "create-archive-tar-file" - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} + - func: "assume-aws-test-secrets-role" - command: subprocess.exec params: binary: bash @@ -2202,9 +2140,7 @@ task_groups: - func: "prepare-resources" - func: "fix-absolute-paths" - func: "create-archive-tar-file" - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} + - func: "assume-aws-test-secrets-role" - command: subprocess.exec params: binary: bash diff --git a/.evergreen/javaConfig.bash b/.evergreen/javaConfig.bash deleted file mode 100644 index 0b0c912526..0000000000 --- a/.evergreen/javaConfig.bash +++ /dev/null @@ -1,19 +0,0 @@ -# Java configurations for evergreen - -export JDK8="/opt/java/jdk8" -export JDK11="/opt/java/jdk11" -export JDK17="/opt/java/jdk17" -export JDK21="/opt/java/jdk21" -# note that `JDK21_GRAALVM` is used in `run-graalvm-native-image-app.sh` -# by dynamically constructing the variable name -export JDK21_GRAALVM="/opt/java/jdk21-graalce" - -if [ -d "$JDK17" ]; then - export JAVA_HOME=$JDK17 -fi - -export JAVA_VERSION=${JAVA_VERSION:-17} - -echo "Java Configs:" -echo "Java Home: ${JAVA_HOME}" -echo "Java test version: ${JAVA_VERSION}" diff --git a/.evergreen/publish.sh b/.evergreen/publish.sh index 9a3e9eb405..50b4f83f87 100755 --- a/.evergreen/publish.sh +++ b/.evergreen/publish.sh @@ -8,7 +8,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" RELEASE=${RELEASE:false} diff --git a/.evergreen/run-atlas-data-lake-test.sh b/.evergreen/run-atlas-data-lake-test.sh index 07938018d6..424efa5ba9 100755 --- a/.evergreen/run-atlas-data-lake-test.sh +++ b/.evergreen/run-atlas-data-lake-test.sh @@ -11,7 +11,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running Atlas Data Lake driver tests" diff --git a/.evergreen/run-atlas-search-index-management-tests.sh b/.evergreen/run-atlas-search-index-management-tests.sh index 7ead4aa0b1..784a9b45a0 100755 --- a/.evergreen/run-atlas-search-index-management-tests.sh +++ b/.evergreen/run-atlas-search-index-management-tests.sh @@ -9,7 +9,7 @@ set -o errexit # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" -source "${RELATIVE_DIR_PATH}/javaConfig.bash" +source "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running Atlas Search tests" ./gradlew -version diff --git a/.evergreen/run-atlas-search-tests.sh b/.evergreen/run-atlas-search-tests.sh index f207647825..01f6bc78b4 100755 --- a/.evergreen/run-atlas-search-tests.sh +++ b/.evergreen/run-atlas-search-tests.sh @@ -9,7 +9,7 @@ set -o errexit # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" -source "${RELATIVE_DIR_PATH}/javaConfig.bash" +source "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running Atlas Search tests" ./gradlew -version diff --git a/.evergreen/run-connectivity-tests.sh b/.evergreen/run-connectivity-tests.sh index 405b5864cb..38ccfaaf76 100755 --- a/.evergreen/run-connectivity-tests.sh +++ b/.evergreen/run-connectivity-tests.sh @@ -13,7 +13,7 @@ set -o errexit # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running connectivity tests with Java ${JAVA_VERSION}" diff --git a/.evergreen/run-csfle-aws-from-environment.sh b/.evergreen/run-csfle-aws-from-environment.sh index a80580d194..6e2c4138c6 100755 --- a/.evergreen/run-csfle-aws-from-environment.sh +++ b/.evergreen/run-csfle-aws-from-environment.sh @@ -12,7 +12,8 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" + echo "Running CSFLE AWS from environment tests" ./gradlew -version diff --git a/.evergreen/run-csfle-tests-with-mongocryptd.sh b/.evergreen/run-csfle-tests-with-mongocryptd.sh index c9733e58a8..5e24eea5ce 100755 --- a/.evergreen/run-csfle-tests-with-mongocryptd.sh +++ b/.evergreen/run-csfle-tests-with-mongocryptd.sh @@ -24,7 +24,7 @@ set -o errexit # Exit the script with error if any of the commands fail MONGODB_URI=${MONGODB_URI:-} RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" ############################################ # Functions # diff --git a/.evergreen/run-deployed-lambda-aws-tests.sh b/.evergreen/run-deployed-lambda-aws-tests.sh index 42010aad65..57c23aa7f0 100755 --- a/.evergreen/run-deployed-lambda-aws-tests.sh +++ b/.evergreen/run-deployed-lambda-aws-tests.sh @@ -4,7 +4,7 @@ set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" # compiled outside of lambda workflow. Note "SkipBuild: True" in template.yaml ./gradlew -version diff --git a/.evergreen/run-graalvm-native-image-app.sh b/.evergreen/run-graalvm-native-image-app.sh index ba24ef764a..e39b9b3a17 100755 --- a/.evergreen/run-graalvm-native-image-app.sh +++ b/.evergreen/run-graalvm-native-image-app.sh @@ -7,7 +7,7 @@ set -o errexit readonly RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" -source "${RELATIVE_DIR_PATH}/javaConfig.bash" +source "${RELATIVE_DIR_PATH}/setup-env.bash" echo "MONGODB_URI: ${MONGODB_URI}" echo "JAVA_HOME: ${JAVA_HOME}" diff --git a/.evergreen/run-gssapi-auth-test.sh b/.evergreen/run-gssapi-auth-test.sh index 67bf10b473..aa131daeee 100755 --- a/.evergreen/run-gssapi-auth-test.sh +++ b/.evergreen/run-gssapi-auth-test.sh @@ -17,7 +17,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running GSSAPI authentication tests with login context name '${LOGIN_CONTEXT_NAME}'" echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab diff --git a/.evergreen/run-kms-tls-tests.sh b/.evergreen/run-kms-tls-tests.sh index b9842d2618..3af2fb086d 100755 --- a/.evergreen/run-kms-tls-tests.sh +++ b/.evergreen/run-kms-tls-tests.sh @@ -11,7 +11,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running KMS TLS tests" cp ${JAVA_HOME}/lib/security/cacerts mongo-truststore diff --git a/.evergreen/run-kotlin-tests.sh b/.evergreen/run-kotlin-tests.sh index e240d7bc25..66acf68809 100755 --- a/.evergreen/run-kotlin-tests.sh +++ b/.evergreen/run-kotlin-tests.sh @@ -13,7 +13,7 @@ TOPOLOGY=${TOPOLOGY:-standalone} # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" if [ "$SSL" != "nossl" ]; then diff --git a/.evergreen/run-load-balancer-tests.sh b/.evergreen/run-load-balancer-tests.sh index 2440cd31cc..4ee1510a90 100755 --- a/.evergreen/run-load-balancer-tests.sh +++ b/.evergreen/run-load-balancer-tests.sh @@ -18,7 +18,7 @@ MONGODB_URI=${MONGODB_URI:-} # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" if [ "$SSL" != "nossl" ]; then # We generate the keystore and truststore on every run with the certs in the drivers-tools repo diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index 7d10b80f12..63e4232839 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -39,7 +39,7 @@ fi cd src RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" ./gradlew -version diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index 45c36227a6..e8a376f4a4 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -11,7 +11,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running MONGODB-AWS authentication tests" diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 55b0599fd0..4a9dbae2b9 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -14,7 +14,7 @@ if [ $OIDC_ENV == "test" ]; then source ${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh # java will not need to be installed, but we need to config RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" - source "${RELATIVE_DIR_PATH}/javaConfig.bash" + source "${RELATIVE_DIR_PATH}/setup-env.bash" elif [ $OIDC_ENV == "azure" ]; then source ./env.sh elif [ $OIDC_ENV == "gcp" ]; then diff --git a/.evergreen/run-ocsp-test.sh b/.evergreen/run-ocsp-test.sh index 2c3d67d66a..048935926a 100755 --- a/.evergreen/run-ocsp-test.sh +++ b/.evergreen/run-ocsp-test.sh @@ -10,7 +10,7 @@ set -o errexit # Exit the script with error if any of the commands fail OCSP_MUST_STAPLE=${OCSP_MUST_STAPLE:-} OCSP_TLS_SHOULD_SUCCEED=${OCSP_TLS_SHOULD_SUCCEED:-} RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" ############################################ # Functions # diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh index 76627a967c..472e434813 100755 --- a/.evergreen/run-perf-tests.sh +++ b/.evergreen/run-perf-tests.sh @@ -12,7 +12,7 @@ tar xf single_and_multi_document.tgz cd .. RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" export TEST_PATH="${PROJECT_DIRECTORY}/driver-performance-test-data/" export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json" diff --git a/.evergreen/run-plain-auth-test.sh b/.evergreen/run-plain-auth-test.sh index bfdd1ce6cb..fbc965df4b 100755 --- a/.evergreen/run-plain-auth-test.sh +++ b/.evergreen/run-plain-auth-test.sh @@ -13,7 +13,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running PLAIN authentication tests" diff --git a/.evergreen/run-reactive-streams-tck-tests.sh b/.evergreen/run-reactive-streams-tck-tests.sh index 44511e849d..6bd5e91a4e 100755 --- a/.evergreen/run-reactive-streams-tck-tests.sh +++ b/.evergreen/run-reactive-streams-tck-tests.sh @@ -7,7 +7,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running Reactive Streams TCK tests with Java ${JAVA_VERSION}" diff --git a/.evergreen/run-scala-tests.sh b/.evergreen/run-scala-tests.sh index e82bb5a56c..02fd240d7c 100755 --- a/.evergreen/run-scala-tests.sh +++ b/.evergreen/run-scala-tests.sh @@ -13,7 +13,7 @@ TOPOLOGY=${TOPOLOGY:-standalone} # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" if [ "$SSL" != "nossl" ]; then diff --git a/.evergreen/run-serverless-tests.sh b/.evergreen/run-serverless-tests.sh index 9e65508f84..5e203c20f9 100755 --- a/.evergreen/run-serverless-tests.sh +++ b/.evergreen/run-serverless-tests.sh @@ -15,7 +15,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Running serverless tests with Java ${JAVA_VERSION}" diff --git a/.evergreen/run-socket-tests.sh b/.evergreen/run-socket-tests.sh index b7525f13ce..df215c953e 100755 --- a/.evergreen/run-socket-tests.sh +++ b/.evergreen/run-socket-tests.sh @@ -21,7 +21,7 @@ COMPRESSOR=${COMPRESSOR:-} # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" SOCKET_REGEX='(.*)localhost:([0-9]+)?(.*)' while [[ $MONGODB_URI =~ $SOCKET_REGEX ]]; do diff --git a/.evergreen/run-socks5-tests.sh b/.evergreen/run-socks5-tests.sh index b11460b877..c6cbb812b8 100755 --- a/.evergreen/run-socks5-tests.sh +++ b/.evergreen/run-socks5-tests.sh @@ -25,7 +25,7 @@ if [ "Windows_NT" == "$OS" ]; then fi RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" ############################################ # Functions # @@ -84,4 +84,4 @@ echo "Running Socks5 tests with Java ${JAVA_VERSION} over $SSL for $TOPOLOGY and provision_ssl ./gradlew -version run_socks5_proxy -run_socks5_prose_tests \ No newline at end of file +run_socks5_prose_tests diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 84481f90ce..ea7f9ec6fd 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -45,7 +45,7 @@ if [ "${SSL}" = "ssl" ] && [ "${STREAM_TYPE}" = "netty" ] && [ "${NETTY_SSL_PROV fi RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" ############################################ # Functions # diff --git a/.evergreen/setup-env.bash b/.evergreen/setup-env.bash new file mode 100644 index 0000000000..cae67cd65e --- /dev/null +++ b/.evergreen/setup-env.bash @@ -0,0 +1,51 @@ +# Java configurations for evergreen + +export JDK8="/opt/java/jdk8" +export JDK11="/opt/java/jdk11" +export JDK17="/opt/java/jdk17" +export JDK21="/opt/java/jdk21" +# note that `JDK21_GRAALVM` is used in `run-graalvm-native-image-app.sh` +# by dynamically constructing the variable name +export JDK21_GRAALVM="/opt/java/jdk21-graalce" + +if [ -d "$JDK17" ]; then + export JAVA_HOME=$JDK17 +fi + +export JAVA_VERSION=${JAVA_VERSION:-17} + +echo "Java Configs:" +echo "Java Home: ${JAVA_HOME}" +echo "Java test version: ${JAVA_VERSION}" + +# Rename environment variables for AWS, Azure, and GCP +if [ -f secrets-export.sh ]; then + echo "Renaming secrets env variables" + . secrets-export.sh + + export AWS_ACCESS_KEY_ID=$FLE_AWS_ACCESS_KEY_ID + export AWS_SECRET_ACCESS_KEY=$FLE_AWS_SECRET_ACCESS_KEY + export AWS_DEFAULT_REGION=$FLE_AWS_DEFAULT_REGION + + export AWS_ACCESS_KEY_ID_AWS_KMS_NAMED=$FLE_AWS_KEY2 + export AWS_SECRET_ACCESS_KEY_AWS_KMS_NAMED=$FLE_AWS_SECRET2 + + export AWS_TEMP_ACCESS_KEY_ID=$CSFLE_AWS_TEMP_ACCESS_KEY_ID + export AWS_TEMP_SECRET_ACCESS_KEY=$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY + export AWS_TEMP_SESSION_TOKEN=$CSFLE_AWS_TEMP_SESSION_TOKEN + + export AZURE_CLIENT_ID=$FLE_AZURE_CLIENTID + export AZURE_TENANT_ID=$FLE_AZURE_TENANTID + export AZURE_CLIENT_SECRET=$FLE_AZURE_CLIENTSECRET + + export GCP_EMAIL=$FLE_GCP_EMAIL + export GCP_PRIVATE_KEY=$FLE_GCP_PRIVATEKEY + + # Unset AWS_SESSION_TOKEN if it is empty + if [ -z "$AWS_SESSION_TOKEN" ];then + unset AWS_SESSION_TOKEN + fi + +else + echo "No secrets env variables found to rename" +fi diff --git a/.evergreen/ssdlc-report.sh b/.evergreen/ssdlc-report.sh index b97eff6d8b..56d5957f5a 100755 --- a/.evergreen/ssdlc-report.sh +++ b/.evergreen/ssdlc-report.sh @@ -24,7 +24,7 @@ fi # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" -source "${RELATIVE_DIR_PATH}/javaConfig.bash" +source "${RELATIVE_DIR_PATH}/setup-env.bash" printf "\nCreating SSDLC reports\n" printf "\nProduct name: %s\n" "${PRODUCT_NAME}" diff --git a/.evergreen/static-checks.sh b/.evergreen/static-checks.sh index 8b65b15e9a..1accf5c168 100755 --- a/.evergreen/static-checks.sh +++ b/.evergreen/static-checks.sh @@ -7,7 +7,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" -. "${RELATIVE_DIR_PATH}/javaConfig.bash" +. "${RELATIVE_DIR_PATH}/setup-env.bash" echo "Compiling JVM drivers" diff --git a/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideEncryptionKmsTlsTest.java b/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideEncryptionKmsTlsTest.java index 2aff6a3233..6e0b5957de 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideEncryptionKmsTlsTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideEncryptionKmsTlsTest.java @@ -109,12 +109,14 @@ public void testInvalidKmsCertificate() { }}); }}) .build(); + // See: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/README.md + String endpoint = expectedKmsTlsError == TlsErrorType.EXPIRED ? "mongodb://127.0.0.1:9000" : "mongodb://127.0.0.1:9001"; try (ClientEncryption clientEncryption = getClientEncryption(clientEncryptionSettings)) { clientEncryption.createDataKey("aws", new DataKeyOptions().masterKey( BsonDocument.parse("{" + "region: \"us-east-1\", " + "key: \"arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0\"," - + "endpoint: \"mongodb://127.0.0.1:8000\"}"))); + + "endpoint: \"" + endpoint + "\"}"))); fail(); } catch (MongoClientException e) { assertNotNull(expectedKmsTlsError.getCauseOfExpectedClass(e)); From eda0a568eb86d3f256740f8493b967d5d52d40d0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 3 Jun 2025 11:52:00 +0100 Subject: [PATCH 05/12] Run gradle classes and cache build before running on azure/gcp/k8s Similar to the overall goals of JAVA-5891 --- .evergreen/.evg.yml | 37 +++++++++++++++++++++-------- .evergreen/gradle-cache.sh | 32 +++++++++++++++++++++++++ .evergreen/run-mongodb-oidc-test.sh | 22 ++++++++++++++--- 3 files changed, 78 insertions(+), 13 deletions(-) create mode 100755 .evergreen/gradle-cache.sh diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 4032ed34b4..6951a22e18 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -129,13 +129,21 @@ functions: params: role_arn: ${aws_test_secrets_role} + "gradle-cache": + - command: shell.exec + params: + working_dir: "src" + script: | + export GRADLE_RO_DEP_CACHE="$(pwd)/build/gradle-cache" + .evergreen/gradle-cache.sh + "create-archive-tar-file": - command: shell.exec params: working_dir: "src" script: | echo "Creating archive tar file at ${ARCHIVE_FILE_PATH}" - tar --exclude="**build/" --exclude-vcs -czf "${ARCHIVE_FILE_PATH}" . + tar --exclude-vcs -czf "${ARCHIVE_FILE_PATH}" . echo "Created archive tar file at ${ARCHIVE_FILE_PATH}" "start-mongo-orchestration": @@ -820,7 +828,7 @@ functions: ${PREPARE_SHELL} export K8S_VARIANT=${VARIANT} export K8S_DRIVERS_TAR_FILE=$ARCHIVE_FILE_PATH - export K8S_TEST_CMD="OIDC_ENV=k8s VARIANT=${VARIANT} ./.evergreen/run-mongodb-oidc-test.sh" + export K8S_TEST_CMD="GRADLE_RO_DEP_CACHE='/tmp/test/build/gradle-cache' OIDC_ENV=k8s VARIANT=${VARIANT} ./.evergreen/run-mongodb-oidc-test.sh" bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh source $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/secrets-export.sh @@ -936,6 +944,8 @@ tasks: - .evergreen/run-mongodb-oidc-test.sh - name: "oidc-auth-test-azure-task" + # Might exceed 1 hour of execution. + exec_timeout_secs: 7200 commands: - command: shell.exec params: @@ -948,11 +958,13 @@ tasks: set -o errexit ${PREPARE_SHELL} export AZUREOIDC_DRIVERS_TAR_FILE=$ARCHIVE_FILE_PATH - export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" - tar --exclude="**build/" --exclude-vcs -czf $AZUREOIDC_DRIVERS_TAR_FILE . + export AZUREOIDC_TEST_CMD="GRADLE_RO_DEP_CACHE='/home/azureuser/build/gradle-cache' OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" + tar --exclude-vcs -czf $AZUREOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh - name: "oidc-auth-test-gcp-task" + # Might exceed 1 hour of execution. + exec_timeout_secs: 7200 commands: - command: shell.exec params: @@ -966,8 +978,8 @@ tasks: # Define the command to run on the VM. # Ensure that we source the environment file created for us, set up any other variables we need, # and then run our test suite on the vm. - export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" - tar --exclude="**build/" --exclude-vcs -czf $GCPOIDC_DRIVERS_TAR_FILE . + export GCPOIDC_TEST_CMD="GRADLE_RO_DEP_CACHE='./build/gradle-cache' OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" + tar --exclude-vcs -czf $GCPOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh - name: "oidc-auth-test-k8s-task" @@ -1695,7 +1707,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - tar --exclude="**build/" --exclude-vcs -czf $ARCHIVE_FILE_PATH . + tar --exclude-vcs -czf $ARCHIVE_FILE_PATH . AZUREKMS_SRC=$ARCHIVE_FILE_PATH AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" @@ -1711,7 +1723,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="MONGODB_URI=mongodb://localhost:27017 PROVIDER=azure AZUREKMS_KEY_VAULT_ENDPOINT=${testazurekms_keyvaultendpoint} AZUREKMS_KEY_NAME=${testazurekms_keyname} ./.evergreen/run-fle-on-demand-credential-test.sh" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + AZUREKMS_CMD="GRADLE_RO_DEP_CACHE='/home/azureuser/build/gradle-cache' MONGODB_URI=mongodb://localhost:27017 PROVIDER=azure AZUREKMS_KEY_VAULT_ENDPOINT=${testazurekms_keyvaultendpoint} AZUREKMS_KEY_NAME=${testazurekms_keyname} ./.evergreen/run-fle-on-demand-credential-test.sh" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: "test-socks5-task" tags: [ ] @@ -1970,6 +1982,7 @@ task_groups: - func: "fetch-source" - func: "prepare-resources" - func: "fix-absolute-paths" + - func: "gradle-cache" - func: "create-archive-tar-file" - command: shell.exec params: @@ -2007,6 +2020,7 @@ task_groups: - func: "fetch-source" - func: "prepare-resources" - func: "fix-absolute-paths" + - func: "gradle-cache" - func: "create-archive-tar-file" - func: "assume-aws-test-secrets-role" - command: shell.exec @@ -2082,6 +2096,7 @@ task_groups: - func: "fetch-source" - func: "prepare-resources" - func: "fix-absolute-paths" + - func: "gradle-cache" - func: "create-archive-tar-file" - func: "assume-aws-test-secrets-role" - command: subprocess.exec @@ -2110,6 +2125,7 @@ task_groups: - func: "fetch-source" - func: "prepare-resources" - func: "fix-absolute-paths" + - func: "gradle-cache" - func: "create-archive-tar-file" - func: "assume-aws-test-secrets-role" - command: subprocess.exec @@ -2139,6 +2155,7 @@ task_groups: - func: "fetch-source" - func: "prepare-resources" - func: "fix-absolute-paths" + - func: "gradle-cache" - func: "create-archive-tar-file" - func: "assume-aws-test-secrets-role" - command: subprocess.exec @@ -2192,14 +2209,14 @@ buildvariants: - name: "test-oidc-azure-variant" display_name: "OIDC Auth Azure" - run_on: ubuntu2204-small + run_on: ubuntu2204-large tasks: - name: "test-oidc-azure-task-group" batchtime: 20160 # 14 days - name: "test-oidc-gcp-variant" display_name: "OIDC Auth GCP" - run_on: ubuntu2204-small + run_on: ubuntu2204-large tasks: - name: "test-oidc-gcp-task-group" batchtime: 20160 # 14 days diff --git a/.evergreen/gradle-cache.sh b/.evergreen/gradle-cache.sh new file mode 100755 index 0000000000..2ffa72908f --- /dev/null +++ b/.evergreen/gradle-cache.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -o xtrace # Write all commands first to stderr +set -o errexit # Exit the script with error if any of the commands fail + +############################################ +# Main Program # +############################################ +RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" +. "${RELATIVE_DIR_PATH}/setup-env.bash" + +echo "Enable caching" +echo "org.gradle.caching=true" >> gradle.properties +echo "kotlin.caching.enabled=true" >> gradle.properties + +echo "Compiling JVM drivers" +./gradlew -version +./gradlew classes --parallel + +# Copy the Gradle dependency cache to the gradle read only dependency cache directory. +if [ -n "$GRADLE_RO_DEP_CACHE" ];then + echo "Copying Gradle dependency cache to $GRADLE_RO_DEP_CACHE" + mkdir -p $GRADLE_RO_DEP_CACHE + + # https://docs.gradle.org/current/userguide/dependency_caching.html#sec:cache-copy + # Gradle suggests removing the "*.lock" files and the `gc.properties` file for saving/restoring cache + cp -r $HOME/.gradle/caches/modules-2 "$GRADLE_RO_DEP_CACHE" + find "$GRADLE_RO_DEP_CACHE" -name "*.lock" -type f | xargs rm -f + find "$GRADLE_RO_DEP_CACHE" -name "gc.properties" -type f | xargs rm -f + + echo "Copied Gradle dependency cache to $GRADLE_RO_DEP_CACHE" +fi diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 4a9dbae2b9..4c22b40665 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -49,7 +49,23 @@ TO_REPLACE="mongodb://" REPLACEMENT="mongodb://$OIDC_ADMIN_USER:$OIDC_ADMIN_PWD@" ADMIN_URI=${MONGODB_URI/$TO_REPLACE/$REPLACEMENT} +echo "Running gradle version" +./gradlew -version + +echo "Running gradle classes compile for driver-core" +./gradlew --parallel --stacktrace --info \ + driver-core:compileJava driver-core:compileTestGroovy + +echo "Running gradle classes compile for driver-sync and driver-reactive-streams" +./gradlew --parallel --stacktrace --info \ + driver-sync:classes driver-reactive-streams:classes + +echo "Running OIDC authentication tests against driver-sync" ./gradlew -Dorg.mongodb.test.uri="$ADMIN_URI" \ - --stacktrace --debug --info --no-build-cache driver-core:cleanTest \ - driver-sync:test --tests OidcAuthenticationProseTests --tests UnifiedAuthTest \ - driver-reactive-streams:test --tests OidcAuthenticationAsyncProseTests \ + --stacktrace --debug --info \ + driver-sync:test --tests OidcAuthenticationProseTests --tests UnifiedAuthTest + +echo "Running OIDC authentication tests against driver-reactive-streams" +./gradlew -Dorg.mongodb.test.uri="$ADMIN_URI" \ + --stacktrace --debug --info driver-reactive-streams:test --tests OidcAuthenticationAsyncProseTests + From e3c053d6946d326f452f773f2a5ca7255a0b651c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 3 Jun 2025 14:19:01 +0100 Subject: [PATCH 06/12] Remove --no-build-cache flags in tests --- .evergreen/run-csfle-aws-from-environment.sh | 4 ++-- .evergreen/run-kms-tls-tests.sh | 4 ++-- .evergreen/run-mongodb-aws-test.sh | 2 +- .evergreen/run-mongodb-oidc-test.sh | 1 - 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.evergreen/run-csfle-aws-from-environment.sh b/.evergreen/run-csfle-aws-from-environment.sh index 6e2c4138c6..a3c7b8fa10 100755 --- a/.evergreen/run-csfle-aws-from-environment.sh +++ b/.evergreen/run-csfle-aws-from-environment.sh @@ -22,12 +22,12 @@ export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} ./gradlew --stacktrace --info -Dorg.mongodb.test.uri=${MONGODB_URI} \ - --no-build-cache driver-sync:cleanTest driver-sync:test --tests ClientSideEncryptionAwsCredentialFromEnvironmentTest + driver-sync:cleanTest driver-sync:test --tests ClientSideEncryptionAwsCredentialFromEnvironmentTest first=$? echo $first ./gradlew --stacktrace --info -Dorg.mongodb.test.uri=${MONGODB_URI} \ - --no-build-cache driver-reactive-streams:cleanTest driver-reactive-streams:test --tests ClientSideEncryptionAwsCredentialFromEnvironmentTest + driver-reactive-streams:cleanTest driver-reactive-streams:test --tests ClientSideEncryptionAwsCredentialFromEnvironmentTest second=$? echo $second diff --git a/.evergreen/run-kms-tls-tests.sh b/.evergreen/run-kms-tls-tests.sh index 3af2fb086d..df3a38c0ee 100755 --- a/.evergreen/run-kms-tls-tests.sh +++ b/.evergreen/run-kms-tls-tests.sh @@ -24,13 +24,13 @@ export KMS_TLS_ERROR_TYPE=${KMS_TLS_ERROR_TYPE} ./gradlew --stacktrace --info ${GRADLE_EXTRA_VARS} -Dorg.mongodb.test.uri=${MONGODB_URI} \ -Dorg.mongodb.test.kms.tls.error.type=${KMS_TLS_ERROR_TYPE} \ - --no-build-cache driver-sync:cleanTest driver-sync:test --tests ClientSideEncryptionKmsTlsTest + driver-sync:cleanTest driver-sync:test --tests ClientSideEncryptionKmsTlsTest first=$? echo $first ./gradlew --stacktrace --info ${GRADLE_EXTRA_VARS} -Dorg.mongodb.test.uri=${MONGODB_URI} \ -Dorg.mongodb.test.kms.tls.error.type=${KMS_TLS_ERROR_TYPE} \ - --no-build-cache driver-reactive-streams:cleanTest driver-reactive-streams:test --tests ClientSideEncryptionKmsTlsTest + driver-reactive-streams:cleanTest driver-reactive-streams:test --tests ClientSideEncryptionKmsTlsTest second=$? echo $second diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index e8a376f4a4..3e1e6c3cf5 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -28,4 +28,4 @@ echo "Running tests with Java ${JAVA_VERSION}" # to run cleanTest to ensure that the test actually executes each run ./gradlew -PjavaVersion="${JAVA_VERSION}" -Dorg.mongodb.test.uri="${MONGODB_URI}" \ -Dorg.mongodb.test.aws.credential.provider="${AWS_CREDENTIAL_PROVIDER}" \ ---stacktrace --debug --info --no-build-cache driver-core:cleanTest driver-core:test --tests AwsAuthenticationSpecification +--stacktrace --debug --info driver-core:cleanTest driver-core:test --tests AwsAuthenticationSpecification diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 4c22b40665..82b1472d92 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -68,4 +68,3 @@ echo "Running OIDC authentication tests against driver-sync" echo "Running OIDC authentication tests against driver-reactive-streams" ./gradlew -Dorg.mongodb.test.uri="$ADMIN_URI" \ --stacktrace --debug --info driver-reactive-streams:test --tests OidcAuthenticationAsyncProseTests - From 54548e8fdc1de494b87a9a0b9a421b4e6deb5d3c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 5 Jun 2025 15:05:28 +0100 Subject: [PATCH 07/12] Disabled oidc azure and gcp tasks JAVA-5896 --- .evergreen/.evg.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 6951a22e18..0f26259450 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -946,6 +946,7 @@ tasks: - name: "oidc-auth-test-azure-task" # Might exceed 1 hour of execution. exec_timeout_secs: 7200 + disable: true # TODO JAVA-5896 commands: - command: shell.exec params: @@ -965,6 +966,7 @@ tasks: - name: "oidc-auth-test-gcp-task" # Might exceed 1 hour of execution. exec_timeout_secs: 7200 + disable: true # TODO JAVA-5896 commands: - command: shell.exec params: From 27c66763ae36d207ed2d5399f4315cc54d82872e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 5 Jun 2025 16:03:23 +0100 Subject: [PATCH 08/12] Disable serverless evergreen task JAVA-5849 --- .evergreen/.evg.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 0f26259450..cd7b0ba963 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -1000,6 +1000,7 @@ tasks: VARIANT: gke - name: "serverless-test-task" + disable: true # TODO JAVA-5849 commands: - func: "assume-aws-test-secrets-role" - func: "start-serverless" From 6ef1debbb60c1ad9013ab91c1ff234a9c0aaf6c3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 5 Jun 2025 16:03:23 +0100 Subject: [PATCH 09/12] Disable serverless evergreen task JAVA-5849 --- .evergreen/.evg.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index cd7b0ba963..0c1c99cf4c 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -575,7 +575,7 @@ functions: ${PREPARE_SHELL} JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh assume-role - "run-aws-auth-test-with-aws-EC2 credentials": + "run-aws-auth-test-with-aws-EC2-credentials": - command: shell.exec type: "test" params: @@ -1071,7 +1071,7 @@ tasks: TOPOLOGY: "server" - func: "assume-aws-test-secrets-role" - func: "add-aws-auth-variables-to-file" - - func: "run-aws-auth-test-with-aws-EC2 credentials" + - func: "run-aws-auth-test-with-aws-EC2-credentials" - name: "aws-auth-test-with-web-identity-credentials-task" commands: From 09f8c4b5cadb3cf8ba1d972c4adf3c560dd4728b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jun 2025 09:04:54 +0100 Subject: [PATCH 10/12] Fix hypen - machine sizes to modern sizes --- .evergreen/.evg.yml | 25 +++++++++++-------- .../run-fle-on-demand-credential-test.sh | 13 ++++++++-- .evergreen/run-mongodb-oidc-test.sh | 4 --- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 0c1c99cf4c..90e1c5e46c 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -999,6 +999,7 @@ tasks: vars: VARIANT: gke + - name: "serverless-test-task" disable: true # TODO JAVA-5849 commands: @@ -1698,6 +1699,8 @@ tasks: GCPKMS_CMD="MONGODB_URI=mongodb://localhost:27017 PROVIDER=gcp ./.evergreen/run-fle-on-demand-credential-test.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "test-azure-kms-task" + # Might exceed 1 hour of execution. + exec_timeout_secs: 7200 commands: - command: shell.exec type: "setup" @@ -1783,7 +1786,7 @@ axes: run_on: "rhel80-small" - id: "ubuntu" display_name: "Ubuntu" - run_on: "ubuntu2004-small" + run_on: "ubuntu2204-small" - id: "topology" display_name: "Topology" @@ -2212,14 +2215,14 @@ buildvariants: - name: "test-oidc-azure-variant" display_name: "OIDC Auth Azure" - run_on: ubuntu2204-large + run_on: ubuntu2204-small tasks: - name: "test-oidc-azure-task-group" batchtime: 20160 # 14 days - name: "test-oidc-gcp-variant" display_name: "OIDC Auth GCP" - run_on: ubuntu2204-large + run_on: ubuntu2204-small tasks: - name: "test-oidc-gcp-task-group" batchtime: 20160 # 14 days @@ -2241,7 +2244,7 @@ buildvariants: - name: "atlas-data-lake-test" display_name: "Atlas Data Lake test" - run_on: "ubuntu2004-small" + run_on: "ubuntu2204-small" tasks: - name: "atlas-data-lake-task" @@ -2253,20 +2256,20 @@ buildvariants: - name: "publish-snapshot" display_name: "Publish Snapshot" - run_on: "ubuntu2004-small" + run_on: "ubuntu2204-small" tasks: - name: "publish-snapshot-task" - name: "publish-release" display_name: "Publish Release" - run_on: "ubuntu2004-small" + run_on: "ubuntu2204-small" tasks: - name: "publish-release-task" - name: "test-gcp-kms-variant" display_name: "GCP KMS" run_on: - - debian11-small + - debian12-small tasks: - name: "test-gcp-kms-task-group" batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README @@ -2274,7 +2277,7 @@ buildvariants: - name: "test-azure-kms-variant" display_name: "Azure KMS" run_on: - - debian11-small + - debian12-small tasks: - name: "test-azure-kms-task-group" batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README @@ -2432,14 +2435,14 @@ buildvariants: matrix_spec: { ssl: "nossl", jdk: [ "jdk8", "jdk17", "jdk21" ], version: [ "4.4", "5.0", "6.0", "7.0", "8.0", "latest" ], os: "ubuntu", aws-credential-provider: "*" } display_name: "MONGODB-AWS Basic Auth test ${version} ${jdk} ${aws-credential-provider}" - run_on: "ubuntu2004-small" + run_on: "ubuntu2204-small" tasks: - name: "aws-auth-test-with-regular-aws-credentials-task" - matrix_name: "aws-ec2-auth-test" matrix_spec: { ssl: "nossl", jdk: [ "jdk21" ], version: [ "7.0" ], os: "ubuntu", aws-credential-provider: "*" } display_name: "MONGODB-AWS Advanced Auth test ${version} ${jdk} ${aws-credential-provider}" - run_on: "ubuntu2004-small" + run_on: "ubuntu2204-small" tasks: - name: "aws-auth-test-with-aws-EC2-credentials-task" - name: "aws-auth-test-with-assume-role-credentials-task" @@ -2451,7 +2454,7 @@ buildvariants: matrix_spec: { ssl: "nossl", auth: "noauth", jdk: "jdk21", version: [ "5.0", "6.0", "7.0", "8.0", "latest" ], topology: "standalone", os: "linux" } display_name: "Accept API Version 2 ${version}" - run_on: "ubuntu2004-small" + run_on: "ubuntu2204-small" tasks: - name: "accept-api-version-2-test-task" diff --git a/.evergreen/run-fle-on-demand-credential-test.sh b/.evergreen/run-fle-on-demand-credential-test.sh index df70ef67cb..4e49bd8275 100755 --- a/.evergreen/run-fle-on-demand-credential-test.sh +++ b/.evergreen/run-fle-on-demand-credential-test.sh @@ -22,15 +22,24 @@ fi export PROVIDER=${PROVIDER} +echo "Running gradle version" +./gradlew -version + +echo "Running gradle classes compile for driver-sync and driver-reactive-streams" +./gradlew --parallel --build-cache --stacktrace --info \ + driver-sync:classes driver-reactive-streams:classes + +echo "Running driver-sync tests" ./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" \ -Dorg.mongodb.test.fle.on.demand.credential.test.success.enabled=true \ - --stacktrace --debug --info driver-sync:test --tests ClientSideEncryptionOnDemandCredentialsTest + --build-cache--stacktrace --info driver-sync:test --tests ClientSideEncryptionOnDemandCredentialsTest first=$? echo $first +echo "Running driver-reactive-streams tests" ./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" \ -Dorg.mongodb.test.fle.on.demand.credential.test.success.enabled=true \ - --stacktrace --debug --info driver-reactive-streams:test --tests ClientSideEncryptionOnDemandCredentialsTest + --build-cache --stacktrace --info driver-reactive-streams:test --tests ClientSideEncryptionOnDemandCredentialsTest second=$? echo $second diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 82b1472d92..92256992de 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -52,10 +52,6 @@ ADMIN_URI=${MONGODB_URI/$TO_REPLACE/$REPLACEMENT} echo "Running gradle version" ./gradlew -version -echo "Running gradle classes compile for driver-core" -./gradlew --parallel --stacktrace --info \ - driver-core:compileJava driver-core:compileTestGroovy - echo "Running gradle classes compile for driver-sync and driver-reactive-streams" ./gradlew --parallel --stacktrace --info \ driver-sync:classes driver-reactive-streams:classes From 7cd24ea6fa9e90b6f08436ff71d8552a2baef5d5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jun 2025 14:56:10 +0100 Subject: [PATCH 11/12] Set machine sizes for gcp and azure JAVA-5896 --- .evergreen/.evg.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 90e1c5e46c..05912e26d4 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -946,7 +946,6 @@ tasks: - name: "oidc-auth-test-azure-task" # Might exceed 1 hour of execution. exec_timeout_secs: 7200 - disable: true # TODO JAVA-5896 commands: - command: shell.exec params: @@ -966,7 +965,6 @@ tasks: - name: "oidc-auth-test-gcp-task" # Might exceed 1 hour of execution. exec_timeout_secs: 7200 - disable: true # TODO JAVA-5896 commands: - command: shell.exec params: @@ -2048,6 +2046,7 @@ task_groups: export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey export AZUREKMS_SCOPE=${testazurekms_scope} export AZUREKMS_VMNAME_PREFIX=JAVADRIVER + export AZUREKMS_MACHINESIZE="Standard_DS3_v2" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - command: expansions.update params: @@ -2111,6 +2110,7 @@ task_groups: include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] env: AZUREOIDC_VMNAME_PREFIX: "JAVA_DRIVER" + AZUREKMS_MACHINESIZE: "Standard_DS3_v2" args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/create-and-setup-vm.sh teardown_task: @@ -2140,7 +2140,7 @@ task_groups: include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] env: GCPOIDC_VMNAME_PREFIX: "JAVA_DRIVER" - GCPKMS_MACHINEtype: "e2-medium" # comparable elapsed time to Azure; default was starved, caused timeouts" + GCPKMS_MACHINETYPE: "e2-standard-4" args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/setup.sh teardown_task: @@ -2269,7 +2269,7 @@ buildvariants: - name: "test-gcp-kms-variant" display_name: "GCP KMS" run_on: - - debian12-small + - ubuntu2204-small tasks: - name: "test-gcp-kms-task-group" batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README @@ -2277,7 +2277,7 @@ buildvariants: - name: "test-azure-kms-variant" display_name: "Azure KMS" run_on: - - debian12-small + - ubuntu2204-small tasks: - name: "test-azure-kms-task-group" batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README From 64cc9c7a8fb341c3f8bcddbb656f09ba898c3f28 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jun 2025 15:02:50 +0100 Subject: [PATCH 12/12] Include extra logging in shell script --- .evergreen/run-fle-on-demand-credential-test.sh | 2 +- .evergreen/run-mongodb-oidc-test.sh | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.evergreen/run-fle-on-demand-credential-test.sh b/.evergreen/run-fle-on-demand-credential-test.sh index 4e49bd8275..6445b53c66 100755 --- a/.evergreen/run-fle-on-demand-credential-test.sh +++ b/.evergreen/run-fle-on-demand-credential-test.sh @@ -32,7 +32,7 @@ echo "Running gradle classes compile for driver-sync and driver-reactive-streams echo "Running driver-sync tests" ./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" \ -Dorg.mongodb.test.fle.on.demand.credential.test.success.enabled=true \ - --build-cache--stacktrace --info driver-sync:test --tests ClientSideEncryptionOnDemandCredentialsTest + --build-cache --stacktrace --info driver-sync:test --tests ClientSideEncryptionOnDemandCredentialsTest first=$? echo $first diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 92256992de..778b8962c0 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -5,7 +5,7 @@ set -eu echo "Running MONGODB-OIDC authentication tests" echo "OIDC_ENV $OIDC_ENV" - +FULL_DESCRIPTION=$OIDC_ENV if [ $OIDC_ENV == "test" ]; then if [ -z "$DRIVERS_TOOLS" ]; then echo "Must specify DRIVERS_TOOLS" @@ -27,6 +27,7 @@ elif [ $OIDC_ENV == "k8s" ]; then exit 1 fi + FULL_DESCRIPTION="${OIDC_ENV} - ${K8S_VARIANT}" # fix for git permissions issue: git config --global --add safe.directory /tmp/test else @@ -34,7 +35,6 @@ else exit 1 fi - if ! which java ; then echo "Installing java..." sudo apt install openjdk-17-jdk -y @@ -52,15 +52,15 @@ ADMIN_URI=${MONGODB_URI/$TO_REPLACE/$REPLACEMENT} echo "Running gradle version" ./gradlew -version -echo "Running gradle classes compile for driver-sync and driver-reactive-streams" +echo "Running gradle classes compile for driver-sync and driver-reactive-streams: ${FULL_DESCRIPTION}" ./gradlew --parallel --stacktrace --info \ driver-sync:classes driver-reactive-streams:classes -echo "Running OIDC authentication tests against driver-sync" +echo "Running OIDC authentication tests against driver-sync: ${FULL_DESCRIPTION}" ./gradlew -Dorg.mongodb.test.uri="$ADMIN_URI" \ --stacktrace --debug --info \ driver-sync:test --tests OidcAuthenticationProseTests --tests UnifiedAuthTest -echo "Running OIDC authentication tests against driver-reactive-streams" +echo "Running OIDC authentication tests against driver-reactive-streams: ${FULL_DESCRIPTION}" ./gradlew -Dorg.mongodb.test.uri="$ADMIN_URI" \ --stacktrace --debug --info driver-reactive-streams:test --tests OidcAuthenticationAsyncProseTests