From 20d345c1c6530a49b4a7931f05564b7f6eee0bea Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 19:04:39 -0500 Subject: [PATCH 01/27] GODRIVER-3367 Use subprocess.exec for Evergreen functions --- .evergreen/config.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 91bbe7e0bd..2bf21a7790 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -172,14 +172,16 @@ functions: role_arn: ${aws_test_secrets_role} run-oidc-auth-test-with-test-credentials: - - command: shell.exec + - command: subprocess.exec type: test params: - working_dir: src/go.mongodb.org/mongo-driver - shell: bash + binary: bash + env: + OIDC: oidc include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - OIDC="oidc" ${RUN_TASK} test-oidc + args: + - ${RUN_TASK} + - test-oidc run-task: - command: shell.exec From 58595ca8da03e761c79500a54128f07b8684ba6a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 19:53:19 -0500 Subject: [PATCH 02/27] update more funcs --- .evergreen/config.yml | 352 ++++++++++++++++++++---------------------- Taskfile.yml | 2 + etc/pr-task.sh | 30 ++++ 3 files changed, 198 insertions(+), 186 deletions(-) create mode 100755 etc/pr-task.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 2bf21a7790..c7edeecee1 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -116,34 +116,26 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh bootstrap-mongo-orchestration: - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - set -x - MONGODB_VERSION=${VERSION} \ - TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - LOAD_BALANCER=${LOAD_BALANCER} \ - bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + binary: "bash" + env: + MONGODB_VERSION: ${VERSION} + include_expansions_in_env: [TOPOLOGY, AUTH, SSL, ORCHESTRATION_FILE, + REQUIRE_API_VERSION, LOAD_BALANCER] + args: ["${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh"] - command: expansions.update params: file: mo-expansion.yml ocsp-bootstrap-mongo-orchestration: - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - MONGODB_VERSION=${VERSION} \ - TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ - bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + binary: "bash" + env: + MONGODB_VERSION: ${VERSION} + include_expansions_in_env: [TOPOLOGY, AUTH, SSL, ORCHESTRATION_FILE] + args: ["${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh"] - command: expansions.update params: file: mo-expansion.yml @@ -179,9 +171,7 @@ functions: env: OIDC: oidc include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - args: - - ${RUN_TASK} - - test-oidc + args: ["${RUN_TASK}", test-oidc] run-task: - command: shell.exec @@ -211,48 +201,44 @@ functions: fi create-api-report: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - export BASE_SHA=${revision} - export HEAD_SHA=${github_commit} - ${RUN_TASK} api-report + binary: "bash" + env: + BASE_SHA: ${revision} + HEAD_SHA: ${github_commit} + args: ["${RUN_TASK}", api-report] "add PR labels": - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - export CONFIG=${PROJECT_DIRECTORY}/.github/labeler.yml - export SCRIPT="${DRIVERS_TOOLS}/.evergreen/github_app/apply-labels.sh" - bash $SCRIPT -l $CONFIG -h ${github_commit} -o "mongodb" -n "mongo-go-driver" + binary: "bash" + env: + COMMIT: ${github_commit} + PR_TASK: "apply-labels" + args: ["${RUN_TASK}", pr-task] "add PR reviewer": - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - export CONFIG=${PROJECT_DIRECTORY}/.github/reviewers.txt - export SCRIPT="${DRIVERS_TOOLS}/.evergreen/github_app/assign-reviewer.sh" - bash $SCRIPT -p $CONFIG -h ${github_commit} -o "mongodb" -n "mongo-go-driver" + binary: "bash" + env: + COMMIT: ${github_commit} + PR_TASK: "assign-reviewer" + args: ["${RUN_TASK}", pr-task] "backport pr": - command: subprocess.exec type: test params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/github_app/backport-pr.sh - - mongodb - - mongo-go-driver - - ${github_commit} + binary: "bash" + env: + COMMIT: ${github_commit} + PR_TASK: "backport-pr" + args: ["${RUN_TASK}", pr-task] send-perf-data: - command: perf.send @@ -263,43 +249,49 @@ functions: - command: ec2.assume_role params: role_arn: "${aws_test_secrets_role}" - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver + binary: "bash" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - ${RUN_TASK} setup-test -- enterprise-plain - ${RUN_TASK} --silent evg-test-enterprise-auth + args: ["${RUN_TASK}", setup-test, --, enterprise-plain] + - command: subprocess.exec + type: test + params: + binary: "bash" + args: ["${RUN_TASK}", --silent, evg-test-enterprise-auth] run-enterprise-gssapi-auth-tests: - command: ec2.assume_role params: role_arn: "${aws_test_secrets_role}" - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver + binary: "bash" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - ${RUN_TASK} setup-test -- enterprise-gssapi - ${RUN_TASK} --silent evg-test-enterprise-auth + args: ["${RUN_TASK}", setup-test, --, enterprise-gssapi] + - command: subprocess.exec + type: test + params: + binary: "bash" + args: ["${RUN_TASK}", --silent, evg-test-enterprise-auth] run-atlas-test: - command: ec2.assume_role params: role_arn: "${aws_test_secrets_role}" - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver + binary: "bash" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - ${RUN_TASK} setup-test -- atlas-connect - ${RUN_TASK} test-atlas-connect + args: ["${RUN_TASK}", setup-test, --, atlas-connect] + - command: subprocess.exec + type: test + params: + binary: "bash" + args: ["${RUN_TASK}", test-atlas-connect] run-ocsp-test: - command: shell.exec @@ -321,24 +313,21 @@ functions: ${RUN_TASK} evg-test-ocsp run-versioned-api-test: - - command: shell.exec + - command: subprocess.exec type: test params: shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - AUTH="${AUTH}" \ - SSL="${SSL}" \ - MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY="${TOPOLOGY}" \ - MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \ - GO_BUILD_TAGS="cse" \ - REQUIRE_API_VERSION="${REQUIRE_API_VERSION}" \ - SKIP_CRYPT_SHARED_LIB=${SKIP_CRYPT_SHARED_LIB} \ - CRYPT_SHARED_LIB_PATH="$CRYPT_SHARED_LIB_PATH" \ - ${RUN_TASK} setup-test - - ${RUN_TASK} evg-test-versioned-api + env: + GO_BUILD_TAGS: "cse" + include_expansions_in_env: ["AUTH", "SSL", "MONGODB_URI", "TOPOLOGY", + "MONGO_GO_DRIVER_COMPRESSOR", "REQUIRE_API_VERSION", + "SKIP_CRYPT_SHARED_LIB", "CRYPT_SHARED_LIB_PATH"] + args: ["${RUN_TASK}", setup-test] + - command: subprocess.exec + type: test + params: + binary: "bash" + args: ["${RUN_TASK}", evg-test-versioned-api] run-load-balancer-tests: - command: shell.exec @@ -372,39 +361,47 @@ functions: ${RUN_TASK} evg-test-load-balancers run-serverless-tests: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${RUN_TASK} setup-test -- serverless - ${RUN_TASK} evg-test-serverless + binary: "bash" + args: ["${RUN_TASK}", setup-test, --, serverless] + - command: subprocess.exec + type: test + params: + binary: "bash" + args: ["${RUN_TASK}", evg-test-serverless] run-atlas-data-lake-test: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - AUTH="auth" \ - SSL="nossl" \ - TOPOLOGY="server" \ - MONGODB_URI="mongodb://mhuser:pencil@localhost" \ - ${RUN_TASK} setup-test - - ${RUN_TASK} evg-test-atlas-data-lake + binary: "bash" + env: + AUTH: auth + SSL: nossl + TOPOLOGY: server + MONGODB_URI: "mongodb://mhuser:pencil@localhost" + args: ["${RUN_TASK}", setup-test] + - command: subprocess.exec + type: test + params: + binary: "bash" + args: ["${RUN_TASK}", evg-test-atlas-data-lake] run-docker-test: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${RUN_TASK} run-docker - TOPOLOGY=sharded_cluster ${RUN_TASK} run-docker -- test-short + binary: "bash" + args: ["${RUN_TASK}", run-docker] + - command: subprocess.exec + type: test + params: + binary: "bash" + env: + TOPOLOGY: sharded_cluster + args: ["${RUN_TASK}", run-docker, --, test-short] run-valid-ocsp-server: - command: shell.exec @@ -471,24 +468,22 @@ functions: --fault revoked run-load-balancer: - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start + binary: "bash" + include_expansions_in_env: ["MONGODB-AWS"] + args: ["${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh", start] - command: expansions.update params: file: lb-expansion.yml run-search-index-tests: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - TEST_INDEX_URI="${TEST_INDEX_URI}" \ - ${RUN_TASK} evg-test-search-index + binary: "bash" + include_expansions_in_env: ["TEST_INDEX_URI"] + args: ["${RUN_TASK}", evg-test-search-index] add-aws-auth-variables-to-file: - command: ec2.assume_role @@ -503,22 +498,18 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh run-aws-auth-test-with-regular-aws-credentials: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${RUN_TASK} evg-test-aws -- regular + binary: "bash" + args: ["${RUN_TASK}", evg-test-aws, --, regular] run-aws-auth-test-with-assume-role-credentials: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${RUN_TASK} evg-test-aws -- assume-role + binary: "bash" + args: ["${RUN_TASK}", evg-test-aws, --, assume-role] run-aws-auth-test-with-aws-EC2-credentials: - command: shell.exec @@ -534,22 +525,18 @@ functions: ${RUN_TASK} evg-test-aws -- ec2 run-aws-auth-test-with-aws-credentials-as-environment-variables: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${RUN_TASK} evg-test-aws -- env-creds + binary: "bash" + args: ["${RUN_TASK}", evg-test-aws, --, env-creds] run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${RUN_TASK} evg-test-aws -- session-creds + binary: "bash" + args: ["${RUN_TASK}", evg-test-aws, --, session-creds] run-aws-ECS-auth-test: - command: subprocess.exec @@ -558,9 +545,7 @@ functions: binary: "bash" working_dir: src include_expansions_in_env: ["SKIP_ECS_AUTH_TEST"] - args: - - ${RUN_TASK} - - evg-test-aws-ecs + args: ["${RUN_TASK}", evg-test-aws-ecs] run-aws-auth-test-with-aws-web-identity-credentials: - command: shell.exec @@ -607,47 +592,47 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh run-kms-tls-test: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - AUTH="${AUTH}" \ - SSL="${SSL}" \ - MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY="${TOPOLOGY}" \ - MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \ - GO_BUILD_TAGS="cse" \ - ${RUN_TASK} setup-test - - KMS_TLS_TESTCASE="${KMS_TLS_TESTCASE}" ${RUN_TASK} evg-test-kms + binary: "bash" + env: + GO_BUILD_TAGS: cse + include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, + MONGO_GO_DRIVER_COMPRESSOR] + args: ["${RUN_TASK}", setup-test] + - command: subprocess.exec + type: test + params: + binary: "bash" + include_expansions_in_env: [KMS_TLS_TESTCASE] + args: ["${RUN_TASK}", evg-test-kms] run-kmip-tests: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - AUTH="${AUTH}" \ - SSL="${SSL}" \ - MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY="${TOPOLOGY}" \ - MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \ - GO_BUILD_TAGS="cse" \ - ${RUN_TASK} setup-test - - KMS_MOCK_SERVERS_RUNNING="true" ${RUN_TASK} evg-test-kmip + binary: "bash" + env: + GO_BUILD_TAGS: cse + include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, + MONGO_GO_DRIVER_COMPRESSOR] + args: ["${RUN_TASK}", setup-test] + - command: subprocess.exec + type: test + params: + binary: "bash" + include_expansions_in_env: [KMS_TLS_TESTCASE] + env: + KMS_MOCK_SERVERS_RUNNING: "true" + args: ["${RUN_TASK}", evg-test-kmip] run-fuzz-tests: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${RUN_TASK} run-fuzz + binary: "bash" + args: ["${RUN_TASK}", run-fuzz] pre: - func: setup-system @@ -1602,26 +1587,23 @@ tasks: - name: "testawskms-task" commands: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${RUN_TASK} test-awskms + binary: "bash" + args: ["${RUN_TASK}", test-awskms] - name: "testawskms-fail-task" # testawskms-fail-task runs without environment variables. # It is expected to fail to obtain credentials. commands: - - command: shell.exec + - command: subprocess.exec type: test params: shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - export EXPECT_ERROR='status=400' - ${RUN_TASK} test-awskms + env: + EXPECT_ERROR: 'status=400' + args: ["${RUN_TASK}", test-awskms] - name: "testazurekms-task" commands: @@ -1681,18 +1663,16 @@ tasks: params: role_arn: ${LAMBDA_AWS_ROLE_ARN} duration_seconds: 3600 - - command: shell.exec + - command: subprocess.exec type: test params: - working_dir: src/go.mongodb.org/mongo-driver - shell: bash + binary: bash env: TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/internal/cmd/faas/awslambda LAMBDA_STACK_NAME: dbx-go-lambda AWS_REGION: us-east-1 include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - ${RUN_TASK} evg-test-deployed-lambda-aws + args: ["${RUN_TASK}", evg-test-deployed-lambda-aws] - name: "oidc-auth-test" commands: diff --git a/Taskfile.yml b/Taskfile.yml index 76c89600e1..17770627e4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -61,6 +61,8 @@ tasks: cherry-picker: bash etc/cherry-picker.sh + pr-task: bash etc/pr-task.sh + # Lint with various GOOS and GOARCH tasks to catch static analysis failures that may only affect # specific operating systems or architectures. For example, staticcheck will only check for 64-bit # alignment of atomically accessed variables on 32-bit architectures (see diff --git a/etc/pr-task.sh b/etc/pr-task.sh new file mode 100755 index 0000000000..c574711b8b --- /dev/null +++ b/etc/pr-task.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -eu + +VARLIST=( + PR_TASK + COMMIT + DRIVERS_TOOLS +) + +# Ensure that all variables required to run the test are set, otherwise throw +# an error. +for VARNAME in "${VARLIST[@]}"; do + [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; +done + +case $PR_TASK in + apply-labels) + CONFIG=$(pwd)/.github/labeler.yml + SCRIPT="${DRIVERS_TOOLS}/.evergreen/github_app/apply-labels.sh" + bash $SCRIPT -l $CONFIG -h $COMMIT -o "mongodb" -n "mongo-go-driver" + ;; + assign-reviewer) + CONFIG=$(pwd)/.github/reviewers.txt + SCRIPT="${DRIVERS_TOOLS}/.evergreen/github_app/assign-reviewer.sh" + bash $SCRIPT -p $CONFIG -h $COMMIT -o "mongodb" -n "mongo-go-driver" + ;; + backport-pr) + bash ${DRIVERS_TOOLS}/.evergreen/github_app/backport-pr.sh mongodb mongo-go-driver $COMMIT + ;; +esac From fd4bc374a55119b8f597aca9f9adda83a6977923 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 21:16:28 -0500 Subject: [PATCH 03/27] fixups --- .evergreen/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c7edeecee1..87f007006b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -316,7 +316,7 @@ functions: - command: subprocess.exec type: test params: - shell: "bash" + binary: "bash" env: GO_BUILD_TAGS: "cse" include_expansions_in_env: ["AUTH", "SSL", "MONGODB_URI", "TOPOLOGY", @@ -471,7 +471,7 @@ functions: - command: subprocess.exec params: binary: "bash" - include_expansions_in_env: ["MONGODB-AWS"] + include_expansions_in_env: ["MONGODB-AWS", "MONGODB_URI"] args: ["${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh", start] - command: expansions.update params: @@ -1600,7 +1600,7 @@ tasks: - command: subprocess.exec type: test params: - shell: "bash" + binary: "bash" env: EXPECT_ERROR: 'status=400' args: ["${RUN_TASK}", test-awskms] From 5f27c264fa495ec117d906288049806383d67de6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 06:26:45 -0500 Subject: [PATCH 04/27] more cleanup --- .evergreen/config.yml | 208 ++++++++++++------------------------ etc/run-mongodb-aws-test.sh | 10 ++ etc/setup-test.sh | 33 +++++- 3 files changed, 112 insertions(+), 139 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 87f007006b..5b88891357 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -20,11 +20,11 @@ exec_timeout_secs: 3600 # What to do when evergreen hits the timeout (`post:` tasks are run automatically) timeout: - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - ls -la + binary: bash + args: [ls, -la] + functions: setup-system: # Executes clone and applies the submitted patch, if any @@ -101,24 +101,24 @@ functions: bucket: ${aws_bucket} permissions: public-read content_type: ${content_type|text/plain} - display_name: "test_suite.tgz" + display_name: test_suite.tgz bootstrap-mongohoused: - command: subprocess.exec params: - binary: "bash" + binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh - command: subprocess.exec params: - binary: "bash" + binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh bootstrap-mongo-orchestration: - command: subprocess.exec params: - binary: "bash" + binary: bash env: MONGODB_VERSION: ${VERSION} include_expansions_in_env: [TOPOLOGY, AUTH, SSL, ORCHESTRATION_FILE, @@ -131,7 +131,7 @@ functions: ocsp-bootstrap-mongo-orchestration: - command: subprocess.exec params: - binary: "bash" + binary: bash env: MONGODB_VERSION: ${VERSION} include_expansions_in_env: [TOPOLOGY, AUTH, SSL, ORCHESTRATION_FILE] @@ -143,18 +143,18 @@ functions: teardown: - command: subprocess.exec params: - binary: "bash" + binary: bash args: # Ensure the instance profile is reassigned for aws tests. - ${DRIVERS_TOOLS}/.evergreen/auth_aws/teardown.sh - command: subprocess.exec params: - binary: "bash" + binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh - command: subprocess.exec params: - binary: "bash" + binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/teardown.sh @@ -170,7 +170,7 @@ functions: binary: bash env: OIDC: oidc - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: [DRIVERS_TOOLS, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] args: ["${RUN_TASK}", test-oidc] run-task: @@ -204,40 +204,40 @@ functions: - command: subprocess.exec type: test params: - binary: "bash" + binary: bash env: - BASE_SHA: ${revision} - HEAD_SHA: ${github_commit} + BASE_SHA: "${revision}" + HEAD_SHA: "${github_commit}" args: ["${RUN_TASK}", api-report] "add PR labels": - command: subprocess.exec type: test params: - binary: "bash" + binary: bash env: - COMMIT: ${github_commit} - PR_TASK: "apply-labels" + COMMIT: "${github_commit}" + PR_TASK: apply-labels args: ["${RUN_TASK}", pr-task] "add PR reviewer": - command: subprocess.exec type: test params: - binary: "bash" + binary: bash env: - COMMIT: ${github_commit} - PR_TASK: "assign-reviewer" + COMMIT: "${github_commit}" + PR_TASK: assign-reviewer args: ["${RUN_TASK}", pr-task] "backport pr": - command: subprocess.exec type: test params: - binary: "bash" + binary: bash env: - COMMIT: ${github_commit} - PR_TASK: "backport-pr" + COMMIT: "${github_commit}" + PR_TASK: backport-pr args: ["${RUN_TASK}", pr-task] send-perf-data: @@ -252,13 +252,13 @@ functions: - command: subprocess.exec type: test params: - binary: "bash" - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + binary: bash + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] args: ["${RUN_TASK}", setup-test, --, enterprise-plain] - command: subprocess.exec type: test params: - binary: "bash" + binary: bash args: ["${RUN_TASK}", --silent, evg-test-enterprise-auth] run-enterprise-gssapi-auth-tests: @@ -268,13 +268,13 @@ functions: - command: subprocess.exec type: test params: - binary: "bash" - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + binary: bash + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] args: ["${RUN_TASK}", setup-test, --, enterprise-gssapi] - command: subprocess.exec type: test params: - binary: "bash" + binary: bash args: ["${RUN_TASK}", --silent, evg-test-enterprise-auth] run-atlas-test: @@ -284,13 +284,13 @@ functions: - command: subprocess.exec type: test params: - binary: "bash" - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + binary: bash + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] args: ["${RUN_TASK}", setup-test, --, atlas-connect] - command: subprocess.exec type: test params: - binary: "bash" + binary: bash args: ["${RUN_TASK}", test-atlas-connect] run-ocsp-test: @@ -316,49 +316,30 @@ functions: - command: subprocess.exec type: test params: - binary: "bash" + binary: bash env: - GO_BUILD_TAGS: "cse" - include_expansions_in_env: ["AUTH", "SSL", "MONGODB_URI", "TOPOLOGY", - "MONGO_GO_DRIVER_COMPRESSOR", "REQUIRE_API_VERSION", - "SKIP_CRYPT_SHARED_LIB", "CRYPT_SHARED_LIB_PATH"] + GO_BUILD_TAGS: cse + include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, MONGO_GO_DRIVER_COMPRESSOR, + REQUIRE_API_VERSION, SKIP_CRYPT_SHARED_LIB, CRYPT_SHARED_LIB_PATH] args: ["${RUN_TASK}", setup-test] - command: subprocess.exec type: test params: - binary: "bash" + binary: bash args: ["${RUN_TASK}", evg-test-versioned-api] run-load-balancer-tests: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - # Verify that the required LB URI expansions are set to ensure that the test runner can correctly connect to - # the LBs. - if [ -z "${SINGLE_MONGOS_LB_URI}" ]; then - echo "SINGLE_MONGOS_LB_URI must be set for testing against LBs" - exit 1 - fi - if [ -z "${MULTI_MONGOS_LB_URI}" ]; then - echo "MULTI_MONGOS_LB_URI must be set for testing against LBs" - exit 1 - fi - - # Per the LB testing spec, the URI of an LB fronting a single mongos should be used to configure internal - # testing Client instances, so we set MONGODB_URI to SINGLE_MONGOS_LB_URI. - AUTH="${AUTH}" \ - SSL="${SSL}" \ - MONGODB_URI="${SINGLE_MONGOS_LB_URI}" \ - TOPOLOGY="${TOPOLOGY}" \ - MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \ - ${RUN_TASK} setup-test - - SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \ - MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \ - ${RUN_TASK} evg-test-load-balancers + binary: bash + include_expansions_in_env: [SINGLE_MONGOS_LB_URI, MULTI_MONGOS_LB_URI, AUTH, SSL, MONGO_GO_DRIVER_COMPRESSOR] + args: ["${RUN_TASK}", setup-test, --, load-balancer] + - command: subprocess.exec + type: test + params: + binary: bash + args: ["${RUN_TASK}", evg-test-load-balancers] run-serverless-tests: - command: subprocess.exec @@ -482,7 +463,8 @@ functions: type: test params: binary: "bash" - include_expansions_in_env: ["TEST_INDEX_URI"] + env: + TEST_SEARCH_INDEX: "${MONGODB_URI}" args: ["${RUN_TASK}", evg-test-search-index] add-aws-auth-variables-to-file: @@ -512,17 +494,12 @@ functions: args: ["${RUN_TASK}", evg-test-aws, --, assume-role] run-aws-auth-test-with-aws-EC2-credentials: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - if [ "${SKIP_EC2_AUTH_TEST}" = "true" ]; then - echo "This platform does not support the EC2 auth test, skipping..." - exit 0 - fi - ${RUN_TASK} evg-test-aws -- ec2 + binary: bash + include_expansions_in_env: [SKIP_EC2_AUTH_TEST] + args: ["${RUN_TASK}", evg-test-aws, --, ec2] run-aws-auth-test-with-aws-credentials-as-environment-variables: - command: subprocess.exec @@ -544,33 +521,24 @@ functions: params: binary: "bash" working_dir: src - include_expansions_in_env: ["SKIP_ECS_AUTH_TEST"] + include_expansions_in_env: [SKIP_ECS_AUTH_TEST] args: ["${RUN_TASK}", evg-test-aws-ecs] run-aws-auth-test-with-aws-web-identity-credentials: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - if [ "${SKIP_WEB_IDENTITY_AUTH_TEST}" = "true" ]; then - echo "This platform does not support the web identity auth test, skipping..." - exit 0 - fi - ${RUN_TASK} evg-test-aws -- web-identity - - command: shell.exec + binary: bash + include_expansions_in_env: [SKIP_WEB_IDENTITY_AUTH_TEST] + args: ["${RUN_TASK}", evg-test-aws, --, web-identity] + - command: subprocess.exec type: test params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - if [ "${SKIP_WEB_IDENTITY_AUTH_TEST}" = "true" ]; then - echo "This platform does not support the web identity auth test, skipping..." - exit 0 - fi - export AWS_ROLE_SESSION_NAME="test" - ${RUN_TASK} evg-test-aws -- web-identity + binary: bash + env: + AWS_ROLE_SESSION_NAME: test + include_expansions_in_env: [SKIP_WEB_IDENTITY_AUTH_TEST] + args: ["${RUN_TASK}", evg-test-aws, --, web-identity] start-cse-servers: - command: ec2.assume_role @@ -1910,30 +1878,6 @@ task_groups: - command: expansions.update params: file: serverless-expansion.yml - - command: shell.exec - params: - shell: "bash" - script: | - # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to - # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other - # downloaded files. - CRYPT_SHARED_LIB_PATH="$(find $(pwd) -maxdepth 1 -type f \ - -name 'mongo_crypt_v1.so' -o \ - -name 'mongo_crypt_v1.dll' -o \ - -name 'mongo_crypt_v1.dylib')" - - # If we're on Windows, convert the "cygdrive" path to Windows-style paths. - if [ "Windows_NT" = "$OS" ]; then - CRYPT_SHARED_LIB_PATH=$(cygpath -m $CRYPT_SHARED_LIB_PATH) - fi - - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" >> crypt-expansion.yml - - # Load the expansion file to make an evergreen variable with the current unique version - - command: expansions.update - params: - file: crypt-expansion.yml - teardown_group: - command: subprocess.exec params: @@ -1973,14 +1917,13 @@ task_groups: setup_group_timeout_secs: 1800 # 30 minutes setup_group: - func: setup-system - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - export AZUREKMS_VMNAME_PREFIX="GODRIVER" - # Get azurekms credentials from the vault. - . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh + binary: bash + env: + AZUREKMS_VMNAME_PREFIX: GODRIVER + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup.sh teardown_group: - command: subprocess.exec params: @@ -2107,15 +2050,6 @@ task_groups: - command: expansions.update params: file: src/go.mongodb.org/mongo-driver/atlas-expansion.yml - - command: shell.exec - params: - working_dir: src/go.mongodb.org/mongo-driver - shell: bash - script: |- - echo "TEST_INDEX_URI: ${MONGODB_URI}" > atlas-expansion.yml - - command: expansions.update - params: - file: src/go.mongodb.org/mongo-driver/atlas-expansion.yml teardown_group: - command: subprocess.exec params: diff --git a/etc/run-mongodb-aws-test.sh b/etc/run-mongodb-aws-test.sh index a21b5c238f..742a54ad57 100755 --- a/etc/run-mongodb-aws-test.sh +++ b/etc/run-mongodb-aws-test.sh @@ -13,6 +13,16 @@ set -o errexit # Exit the script with error if any of the commands fail echo "Running MONGODB-AWS authentication tests" +if [ $1 == "ec2" ] && [ ${SKIP_EC2_AUTH_TEST:-} == "true" ]; then + echo "This platform does not support the EC2 auth test, skipping..." + exit 0 +fi + +if [ $1 == "web-identity" ] && [ ${SKIP_WEB_IDENTITY_AUTH_TEST:-} == "true" ]; then + echo "This platform does not support the EC2 auth test, skipping..." + exit 0 +fi + # Handle credentials and environment setup. . $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1 diff --git a/etc/setup-test.sh b/etc/setup-test.sh index a1d7263376..5b7bacc41e 100755 --- a/etc/setup-test.sh +++ b/etc/setup-test.sh @@ -39,6 +39,20 @@ case ${1:-} in atlas-connect) . $DRIVERS_TOOLS/.evergreen/secrets_handling/setup-secrets.sh drivers/atlas_connect ;; + load-balancer) + # Verify that the required LB URI expansions are set to ensure that the test runner can correctly connect to + # the LBs. + if [ -z "${SINGLE_MONGOS_LB_URI}" ]; then + echo "SINGLE_MONGOS_LB_URI must be set for testing against LBs" + exit 1 + fi + if [ -z "${MULTI_MONGOS_LB_URI}" ]; then + echo "MULTI_MONGOS_LB_URI must be set for testing against LBs" + exit 1 + fi + MONGODB_URI="${SINGLE_MONGOS_LB_URI}" + LOAD_BALANCER="true" + ;; esac # Handle encryption. @@ -58,9 +72,19 @@ if [[ "${GO_BUILD_TAGS}" =~ cse ]]; then if [ "${SKIP_CRYPT_SHARED_LIB:-''}" = "true" ]; then CRYPT_SHARED_LIB_PATH="" echo "crypt_shared library is skipped" - elif [ -z "${CRYPT_SHARED_LIB_PATH:-}" ]; then - echo "crypt_shared library path is empty" else + # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to + # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other + # downloaded files. + CRYPT_SHARED_LIB_PATH="$(find "$(pwd)" -maxdepth 1 -type f \ + -name 'mongo_crypt_v1.so' -o \ + -name 'mongo_crypt_v1.dll' -o \ + -name 'mongo_crypt_v1.dylib')" + + # If we're on Windows, convert the "cygdrive" path to Windows-style paths. + if [ "Windows_NT" = "$OS" ]; then + CRYPT_SHARED_LIB_PATH=$(cygpath -m $CRYPT_SHARED_LIB_PATH) + fi echo "crypt_shared library will be loaded from path: $CRYPT_SHARED_LIB_PATH" fi fi @@ -116,6 +140,11 @@ if [ -n "${SERVERLESS:-}" ]; then echo "SERVERLESS_ATLAS_PASSWORD=$SERVERLESS_ATLAS_PASSWORD" >> .test.env fi +if [ -n "${LOAD_BALANCER:-}" ];then + echo "SINGLE_MONGOS_LB_URI=${SINGLE_MONGOS_LB_URI}" >> .test.env + echo "MULTI_MONGOS_LB_URI=${MULTI_MONGOS_LB_URI}" >> .test.env +fi + # Add secrets to the test file. if [ -f "secrets-export.sh" ]; then while read p; do From 8ab946bfab556efa02cad6ef667e3f4618776295 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 06:53:28 -0500 Subject: [PATCH 05/27] fix crypt_shared handling --- etc/run-mongodb-aws-test.sh | 4 ++-- etc/setup-test.sh | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/etc/run-mongodb-aws-test.sh b/etc/run-mongodb-aws-test.sh index 742a54ad57..cf24fa9f40 100755 --- a/etc/run-mongodb-aws-test.sh +++ b/etc/run-mongodb-aws-test.sh @@ -13,12 +13,12 @@ set -o errexit # Exit the script with error if any of the commands fail echo "Running MONGODB-AWS authentication tests" -if [ $1 == "ec2" ] && [ ${SKIP_EC2_AUTH_TEST:-} == "true" ]; then +if [ "$1" == "ec2" ] && [ "${SKIP_EC2_AUTH_TEST:-}" == "true" ]; then echo "This platform does not support the EC2 auth test, skipping..." exit 0 fi -if [ $1 == "web-identity" ] && [ ${SKIP_WEB_IDENTITY_AUTH_TEST:-} == "true" ]; then +if [ "$1" == "web-identity" ] && [ "${SKIP_WEB_IDENTITY_AUTH_TEST:-}" == "true" ]; then echo "This platform does not support the EC2 auth test, skipping..." exit 0 fi diff --git a/etc/setup-test.sh b/etc/setup-test.sh index 5b7bacc41e..f416bf7790 100755 --- a/etc/setup-test.sh +++ b/etc/setup-test.sh @@ -65,8 +65,8 @@ if [[ "${GO_BUILD_TAGS}" =~ cse ]]; then LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64 if [ "$(uname -s)" = "Darwin" ]; then - PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig - DYLD_FALLBACK_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib + PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig + DYLD_FALLBACK_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib fi if [ "${SKIP_CRYPT_SHARED_LIB:-''}" = "true" ]; then @@ -76,7 +76,12 @@ if [[ "${GO_BUILD_TAGS}" =~ cse ]]; then # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other # downloaded files. - CRYPT_SHARED_LIB_PATH="$(find "$(pwd)" -maxdepth 1 -type f \ + if [ "Windows_NT" = "$OS" ]; then + BASE_PATH="c:/libmongocrypt" + else + BASE_PATH="$(pwd)/install" + fi + CRYPT_SHARED_LIB_PATH="$(find "$BASE_PATH" -maxdepth 1 -type f \ -name 'mongo_crypt_v1.so' -o \ -name 'mongo_crypt_v1.dll' -o \ -name 'mongo_crypt_v1.dylib')" From 9f5ff3e24778377056d89d09dad35c321cae690e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 07:11:24 -0500 Subject: [PATCH 06/27] fix crypt_shared handling --- etc/setup-test.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/etc/setup-test.sh b/etc/setup-test.sh index f416bf7790..8f0f9aca82 100755 --- a/etc/setup-test.sh +++ b/etc/setup-test.sh @@ -73,15 +73,10 @@ if [[ "${GO_BUILD_TAGS}" =~ cse ]]; then CRYPT_SHARED_LIB_PATH="" echo "crypt_shared library is skipped" else - # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to + # Find the crypt_shared library file and set the CRYPT_SHARED_LIB_PATH to # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other # downloaded files. - if [ "Windows_NT" = "$OS" ]; then - BASE_PATH="c:/libmongocrypt" - else - BASE_PATH="$(pwd)/install" - fi - CRYPT_SHARED_LIB_PATH="$(find "$BASE_PATH" -maxdepth 1 -type f \ + CRYPT_SHARED_LIB_PATH="$(find "$MONGODB_BINARIES" -maxdepth 1 -type f \ -name 'mongo_crypt_v1.so' -o \ -name 'mongo_crypt_v1.dll' -o \ -name 'mongo_crypt_v1.dylib')" From 28e19763fa8cf52bdbaecd0b90e19995ff4cbe74 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 10:10:41 -0500 Subject: [PATCH 07/27] fix crypt_shared handling and docker test --- .evergreen/config.yml | 2 +- .evergreen/setup-system.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5b88891357..35f3f7824f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -375,7 +375,7 @@ functions: type: test params: binary: "bash" - args: ["${RUN_TASK}", run-docker] + args: ["${RUN_TASK}", run-docker, --, test-short] - command: subprocess.exec type: test params: diff --git a/.evergreen/setup-system.sh b/.evergreen/setup-system.sh index 7b7e402da6..4343b5190d 100755 --- a/.evergreen/setup-system.sh +++ b/.evergreen/setup-system.sh @@ -72,6 +72,7 @@ export GOPATH="$GOPATH" export GOCACHE="$GOCACHE" export DRIVERS_TOOLS="$DRIVERS_TOOLS" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" +export MONGODB_BINARIES="$MONGODB_BINARIES" export PATH="$PATH" EOT From 0e96899392eaee8a3b46b9534cd017295cd35160 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 10:27:18 -0500 Subject: [PATCH 08/27] Revert "fix crypt_shared handling and docker test" This reverts commit 28e19763fa8cf52bdbaecd0b90e19995ff4cbe74. --- .evergreen/config.yml | 2 +- .evergreen/setup-system.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 35f3f7824f..5b88891357 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -375,7 +375,7 @@ functions: type: test params: binary: "bash" - args: ["${RUN_TASK}", run-docker, --, test-short] + args: ["${RUN_TASK}", run-docker] - command: subprocess.exec type: test params: diff --git a/.evergreen/setup-system.sh b/.evergreen/setup-system.sh index 4343b5190d..7b7e402da6 100755 --- a/.evergreen/setup-system.sh +++ b/.evergreen/setup-system.sh @@ -72,7 +72,6 @@ export GOPATH="$GOPATH" export GOCACHE="$GOCACHE" export DRIVERS_TOOLS="$DRIVERS_TOOLS" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" -export MONGODB_BINARIES="$MONGODB_BINARIES" export PATH="$PATH" EOT From 46d40ad194d36550ed028be48fba0aaf8149d596 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 10:27:47 -0500 Subject: [PATCH 09/27] Revert "Revert "fix crypt_shared handling and docker test"" This reverts commit 0e96899392eaee8a3b46b9534cd017295cd35160. --- .evergreen/config.yml | 2 +- .evergreen/setup-system.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5b88891357..35f3f7824f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -375,7 +375,7 @@ functions: type: test params: binary: "bash" - args: ["${RUN_TASK}", run-docker] + args: ["${RUN_TASK}", run-docker, --, test-short] - command: subprocess.exec type: test params: diff --git a/.evergreen/setup-system.sh b/.evergreen/setup-system.sh index 7b7e402da6..4343b5190d 100755 --- a/.evergreen/setup-system.sh +++ b/.evergreen/setup-system.sh @@ -72,6 +72,7 @@ export GOPATH="$GOPATH" export GOCACHE="$GOCACHE" export DRIVERS_TOOLS="$DRIVERS_TOOLS" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" +export MONGODB_BINARIES="$MONGODB_BINARIES" export PATH="$PATH" EOT From 1f6161dac78a14fae61c61a4fdf8b44920a10c3a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 14:44:23 -0500 Subject: [PATCH 10/27] fix docker --- .evergreen/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 35f3f7824f..e906260404 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -375,14 +375,15 @@ functions: type: test params: binary: "bash" - args: ["${RUN_TASK}", run-docker, --, test-short] + args: ["${RUN_TASK}", run-docker] - command: subprocess.exec type: test params: binary: "bash" env: TOPOLOGY: sharded_cluster - args: ["${RUN_TASK}", run-docker, --, test-short] + TASKFILE_TARGET: test-short + args: ["${RUN_TASK}", run-docker] run-valid-ocsp-server: - command: shell.exec From 07de58c588788a8018aac932e83df2c794e0fe30 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 14:47:26 -0500 Subject: [PATCH 11/27] fix handling of crypt shared --- etc/setup-test.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/etc/setup-test.sh b/etc/setup-test.sh index 8f0f9aca82..b90d420613 100755 --- a/etc/setup-test.sh +++ b/etc/setup-test.sh @@ -72,19 +72,9 @@ if [[ "${GO_BUILD_TAGS}" =~ cse ]]; then if [ "${SKIP_CRYPT_SHARED_LIB:-''}" = "true" ]; then CRYPT_SHARED_LIB_PATH="" echo "crypt_shared library is skipped" + elif [ -z "${CRYPT_SHARED_LIB_PATH:-}" ]; then + echo "crypt_shared library path is empty" else - # Find the crypt_shared library file and set the CRYPT_SHARED_LIB_PATH to - # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other - # downloaded files. - CRYPT_SHARED_LIB_PATH="$(find "$MONGODB_BINARIES" -maxdepth 1 -type f \ - -name 'mongo_crypt_v1.so' -o \ - -name 'mongo_crypt_v1.dll' -o \ - -name 'mongo_crypt_v1.dylib')" - - # If we're on Windows, convert the "cygdrive" path to Windows-style paths. - if [ "Windows_NT" = "$OS" ]; then - CRYPT_SHARED_LIB_PATH=$(cygpath -m $CRYPT_SHARED_LIB_PATH) - fi echo "crypt_shared library will be loaded from path: $CRYPT_SHARED_LIB_PATH" fi fi From fa5efd36b1f40b9ece4edddc50fb5f709dc65476 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 14:59:51 -0500 Subject: [PATCH 12/27] another docker fix --- .evergreen/config.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e906260404..2520801b37 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -375,6 +375,7 @@ functions: type: test params: binary: "bash" + TASKFILE_TARGET: test-short args: ["${RUN_TASK}", run-docker] - command: subprocess.exec type: test @@ -1879,6 +1880,29 @@ task_groups: - command: expansions.update params: file: serverless-expansion.yml + - command: shell.exec + params: + shell: "bash" + script: | + # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to + # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other + # downloaded files. + CRYPT_SHARED_LIB_PATH="$(find $(pwd) -maxdepth 1 -type f \ + -name 'mongo_crypt_v1.so' -o \ + -name 'mongo_crypt_v1.dll' -o \ + -name 'mongo_crypt_v1.dylib')" + + # If we're on Windows, convert the "cygdrive" path to Windows-style paths. + if [ "Windows_NT" = "$OS" ]; then + CRYPT_SHARED_LIB_PATH=$(cygpath -m $CRYPT_SHARED_LIB_PATH) + fi + + echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" >> crypt-expansion.yml + + # Load the expansion file to make an evergreen variable with the current unique version + - command: expansions.update + params: + file: crypt-expansion.yml teardown_group: - command: subprocess.exec params: From 9925d303ac67353a7837769408451332dfcbb70f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 15:59:35 -0500 Subject: [PATCH 13/27] another docker fix --- .evergreen/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 2520801b37..114a95aac9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -375,7 +375,8 @@ functions: type: test params: binary: "bash" - TASKFILE_TARGET: test-short + env: + TASKFILE_TARGET: test-short args: ["${RUN_TASK}", run-docker] - command: subprocess.exec type: test From 4e51908ab2a9d734c8af7b9cbd16622781d8448e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 16:18:08 -0500 Subject: [PATCH 14/27] another docker fix --- etc/run_docker.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/run_docker.sh b/etc/run_docker.sh index 175fe1239d..5624d9f983 100755 --- a/etc/run_docker.sh +++ b/etc/run_docker.sh @@ -11,7 +11,8 @@ PLATFORM=${DOCKER_PLATFORM:-} docker build $PLATFORM -t go-test . # Handle environment variables and optional positional arg for the taskfile target. -TASKFILE_TARGET=${1:-evg-test-versioned-api} +TASKFILE_TARGET=${TASKFILE_TARGET:-$1} +TASKFILE_TARGET=${TASKFILE_TARGET:-evg-test-versioned-api} GO_BUILD_TAGS=${GO_BUILD_TAGS:-""} ARGS=" -e TASKFILE_TARGET=$TASKFILE_TARGET" From adc90c5de6f3c188ff8ed338c45e2653010fed84 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 16:18:58 -0500 Subject: [PATCH 15/27] another docker fix --- .evergreen/config.yml | 7 ++----- Taskfile.yml | 2 +- etc/run_docker.sh | 3 +-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 114a95aac9..a519f2e398 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -375,17 +375,14 @@ functions: type: test params: binary: "bash" - env: - TASKFILE_TARGET: test-short - args: ["${RUN_TASK}", run-docker] + args: ["${RUN_TASK}", run-docker, --, test-short] - command: subprocess.exec type: test params: binary: "bash" env: TOPOLOGY: sharded_cluster - TASKFILE_TARGET: test-short - args: ["${RUN_TASK}", run-docker] + args: ["${RUN_TASK}", run-docker, --, test-short] run-valid-ocsp-server: - command: shell.exec diff --git a/Taskfile.yml b/Taskfile.yml index 17770627e4..348c9bd0f9 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -55,7 +55,7 @@ tasks: status: - test -d install || test -d /cygdrive/c/libmongocrypt/bin - run-docker: bash etc/run_docker.sh + run-docker: bash etc/run_docker.sh {{.CLI_ARGS}} run-fuzz: bash etc/run-fuzz.sh diff --git a/etc/run_docker.sh b/etc/run_docker.sh index 5624d9f983..175fe1239d 100755 --- a/etc/run_docker.sh +++ b/etc/run_docker.sh @@ -11,8 +11,7 @@ PLATFORM=${DOCKER_PLATFORM:-} docker build $PLATFORM -t go-test . # Handle environment variables and optional positional arg for the taskfile target. -TASKFILE_TARGET=${TASKFILE_TARGET:-$1} -TASKFILE_TARGET=${TASKFILE_TARGET:-evg-test-versioned-api} +TASKFILE_TARGET=${1:-evg-test-versioned-api} GO_BUILD_TAGS=${GO_BUILD_TAGS:-""} ARGS=" -e TASKFILE_TARGET=$TASKFILE_TARGET" From 80390136a0c1167d4ac031c1f860b71011367086 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 16:21:05 -0500 Subject: [PATCH 16/27] test docker --- .evergreen/config.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a519f2e398..35f3f7824f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1878,29 +1878,6 @@ task_groups: - command: expansions.update params: file: serverless-expansion.yml - - command: shell.exec - params: - shell: "bash" - script: | - # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to - # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other - # downloaded files. - CRYPT_SHARED_LIB_PATH="$(find $(pwd) -maxdepth 1 -type f \ - -name 'mongo_crypt_v1.so' -o \ - -name 'mongo_crypt_v1.dll' -o \ - -name 'mongo_crypt_v1.dylib')" - - # If we're on Windows, convert the "cygdrive" path to Windows-style paths. - if [ "Windows_NT" = "$OS" ]; then - CRYPT_SHARED_LIB_PATH=$(cygpath -m $CRYPT_SHARED_LIB_PATH) - fi - - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" >> crypt-expansion.yml - - # Load the expansion file to make an evergreen variable with the current unique version - - command: expansions.update - params: - file: crypt-expansion.yml teardown_group: - command: subprocess.exec params: From 532c3ee505f7677198d6dfcf364765bff98170f8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Oct 2024 18:55:02 -0500 Subject: [PATCH 17/27] fix docker --- etc/docker_entry.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/docker_entry.sh b/etc/docker_entry.sh index 52f4db7eb1..7927971590 100755 --- a/etc/docker_entry.sh +++ b/etc/docker_entry.sh @@ -14,4 +14,4 @@ export PATH="$MONGODB_BINARIES:$HOME/go/bin:$PATH" task setup-test # Run the test. -task evg-test +task $TASKFILE_TARGET From 8dad031100446d61d59d7fa7cebde7e07bf81dd0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 4 Oct 2024 06:12:02 -0500 Subject: [PATCH 18/27] fix message --- etc/run-mongodb-aws-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/run-mongodb-aws-test.sh b/etc/run-mongodb-aws-test.sh index cf24fa9f40..4a74b3c027 100755 --- a/etc/run-mongodb-aws-test.sh +++ b/etc/run-mongodb-aws-test.sh @@ -19,7 +19,7 @@ if [ "$1" == "ec2" ] && [ "${SKIP_EC2_AUTH_TEST:-}" == "true" ]; then fi if [ "$1" == "web-identity" ] && [ "${SKIP_WEB_IDENTITY_AUTH_TEST:-}" == "true" ]; then - echo "This platform does not support the EC2 auth test, skipping..." + echo "This platform does not support the web identity auth test, skipping..." exit 0 fi From 69cc1360ad20ff013d4ab54090e256f24130d9bf Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 4 Oct 2024 06:16:40 -0500 Subject: [PATCH 19/27] remove unused variable --- .evergreen/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 35f3f7824f..a21309c60b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -590,7 +590,6 @@ functions: type: test params: binary: "bash" - include_expansions_in_env: [KMS_TLS_TESTCASE] env: KMS_MOCK_SERVERS_RUNNING: "true" args: ["${RUN_TASK}", evg-test-kmip] From d6d72ab3cc1016db3f3c151f4f077ccc81c6ecda Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 5 Oct 2024 11:23:05 -0500 Subject: [PATCH 20/27] use alias for run-task --- .evergreen/config.yml | 77 ++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a21309c60b..fa2dae30f8 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -41,6 +41,7 @@ functions: GOROOT: ${GO_DIST} IS_PATCH: ${is_patch} VERSION_ID: ${version_id} + RUN_TASK: &run-task src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh args: - ".evergreen/setup-system.sh" - command: expansions.update @@ -171,7 +172,7 @@ functions: env: OIDC: oidc include_expansions_in_env: [DRIVERS_TOOLS, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - args: ["${RUN_TASK}", test-oidc] + args: [*run-task, test-oidc] run-task: - command: shell.exec @@ -208,7 +209,7 @@ functions: env: BASE_SHA: "${revision}" HEAD_SHA: "${github_commit}" - args: ["${RUN_TASK}", api-report] + args: [*run-task, api-report] "add PR labels": - command: subprocess.exec @@ -218,7 +219,7 @@ functions: env: COMMIT: "${github_commit}" PR_TASK: apply-labels - args: ["${RUN_TASK}", pr-task] + args: [*run-task, pr-task] "add PR reviewer": - command: subprocess.exec @@ -228,7 +229,7 @@ functions: env: COMMIT: "${github_commit}" PR_TASK: assign-reviewer - args: ["${RUN_TASK}", pr-task] + args: [*run-task, pr-task] "backport pr": - command: subprocess.exec @@ -238,7 +239,7 @@ functions: env: COMMIT: "${github_commit}" PR_TASK: backport-pr - args: ["${RUN_TASK}", pr-task] + args: [*run-task, pr-task] send-perf-data: - command: perf.send @@ -254,12 +255,12 @@ functions: params: binary: bash include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - args: ["${RUN_TASK}", setup-test, --, enterprise-plain] + args: [*run-task, setup-test, --, enterprise-plain] - command: subprocess.exec type: test params: binary: bash - args: ["${RUN_TASK}", --silent, evg-test-enterprise-auth] + args: [*run-task, --silent, evg-test-enterprise-auth] run-enterprise-gssapi-auth-tests: - command: ec2.assume_role @@ -270,12 +271,12 @@ functions: params: binary: bash include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - args: ["${RUN_TASK}", setup-test, --, enterprise-gssapi] + args: [*run-task, setup-test, --, enterprise-gssapi] - command: subprocess.exec type: test params: binary: bash - args: ["${RUN_TASK}", --silent, evg-test-enterprise-auth] + args: [*run-task, --silent, evg-test-enterprise-auth] run-atlas-test: - command: ec2.assume_role @@ -286,12 +287,12 @@ functions: params: binary: bash include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - args: ["${RUN_TASK}", setup-test, --, atlas-connect] + args: [*run-task, setup-test, --, atlas-connect] - command: subprocess.exec type: test params: binary: bash - args: ["${RUN_TASK}", test-atlas-connect] + args: [*run-task, test-atlas-connect] run-ocsp-test: - command: shell.exec @@ -321,12 +322,12 @@ functions: GO_BUILD_TAGS: cse include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, MONGO_GO_DRIVER_COMPRESSOR, REQUIRE_API_VERSION, SKIP_CRYPT_SHARED_LIB, CRYPT_SHARED_LIB_PATH] - args: ["${RUN_TASK}", setup-test] + args: [*run-task, setup-test] - command: subprocess.exec type: test params: binary: bash - args: ["${RUN_TASK}", evg-test-versioned-api] + args: [*run-task, evg-test-versioned-api] run-load-balancer-tests: - command: subprocess.exec @@ -334,24 +335,24 @@ functions: params: binary: bash include_expansions_in_env: [SINGLE_MONGOS_LB_URI, MULTI_MONGOS_LB_URI, AUTH, SSL, MONGO_GO_DRIVER_COMPRESSOR] - args: ["${RUN_TASK}", setup-test, --, load-balancer] + args: [*run-task, setup-test, --, load-balancer] - command: subprocess.exec type: test params: binary: bash - args: ["${RUN_TASK}", evg-test-load-balancers] + args: [*run-task, evg-test-load-balancers] run-serverless-tests: - command: subprocess.exec type: test params: binary: "bash" - args: ["${RUN_TASK}", setup-test, --, serverless] + args: [*run-task, setup-test, --, serverless] - command: subprocess.exec type: test params: binary: "bash" - args: ["${RUN_TASK}", evg-test-serverless] + args: [*run-task, evg-test-serverless] run-atlas-data-lake-test: - command: subprocess.exec @@ -363,26 +364,26 @@ functions: SSL: nossl TOPOLOGY: server MONGODB_URI: "mongodb://mhuser:pencil@localhost" - args: ["${RUN_TASK}", setup-test] + args: [*run-task, setup-test] - command: subprocess.exec type: test params: binary: "bash" - args: ["${RUN_TASK}", evg-test-atlas-data-lake] + args: [*run-task, evg-test-atlas-data-lake] run-docker-test: - command: subprocess.exec type: test params: binary: "bash" - args: ["${RUN_TASK}", run-docker, --, test-short] + args: [*run-task, run-docker, --, test-short] - command: subprocess.exec type: test params: binary: "bash" env: TOPOLOGY: sharded_cluster - args: ["${RUN_TASK}", run-docker, --, test-short] + args: [*run-task, run-docker, --, test-short] run-valid-ocsp-server: - command: shell.exec @@ -465,7 +466,7 @@ functions: binary: "bash" env: TEST_SEARCH_INDEX: "${MONGODB_URI}" - args: ["${RUN_TASK}", evg-test-search-index] + args: [*run-task, evg-test-search-index] add-aws-auth-variables-to-file: - command: ec2.assume_role @@ -484,14 +485,14 @@ functions: type: test params: binary: "bash" - args: ["${RUN_TASK}", evg-test-aws, --, regular] + args: [*run-task, evg-test-aws, --, regular] run-aws-auth-test-with-assume-role-credentials: - command: subprocess.exec type: test params: binary: "bash" - args: ["${RUN_TASK}", evg-test-aws, --, assume-role] + args: [*run-task, evg-test-aws, --, assume-role] run-aws-auth-test-with-aws-EC2-credentials: - command: subprocess.exec @@ -499,21 +500,21 @@ functions: params: binary: bash include_expansions_in_env: [SKIP_EC2_AUTH_TEST] - args: ["${RUN_TASK}", evg-test-aws, --, ec2] + args: [*run-task, evg-test-aws, --, ec2] run-aws-auth-test-with-aws-credentials-as-environment-variables: - command: subprocess.exec type: test params: binary: "bash" - args: ["${RUN_TASK}", evg-test-aws, --, env-creds] + args: [*run-task, evg-test-aws, --, env-creds] run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables: - command: subprocess.exec type: test params: binary: "bash" - args: ["${RUN_TASK}", evg-test-aws, --, session-creds] + args: [*run-task, evg-test-aws, --, session-creds] run-aws-ECS-auth-test: - command: subprocess.exec @@ -522,7 +523,7 @@ functions: binary: "bash" working_dir: src include_expansions_in_env: [SKIP_ECS_AUTH_TEST] - args: ["${RUN_TASK}", evg-test-aws-ecs] + args: [*run-task, evg-test-aws-ecs] run-aws-auth-test-with-aws-web-identity-credentials: - command: subprocess.exec @@ -530,7 +531,7 @@ functions: params: binary: bash include_expansions_in_env: [SKIP_WEB_IDENTITY_AUTH_TEST] - args: ["${RUN_TASK}", evg-test-aws, --, web-identity] + args: [*run-task, evg-test-aws, --, web-identity] - command: subprocess.exec type: test params: @@ -538,7 +539,7 @@ functions: env: AWS_ROLE_SESSION_NAME: test include_expansions_in_env: [SKIP_WEB_IDENTITY_AUTH_TEST] - args: ["${RUN_TASK}", evg-test-aws, --, web-identity] + args: [*run-task, evg-test-aws, --, web-identity] start-cse-servers: - command: ec2.assume_role @@ -568,13 +569,13 @@ functions: GO_BUILD_TAGS: cse include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, MONGO_GO_DRIVER_COMPRESSOR] - args: ["${RUN_TASK}", setup-test] + args: [*run-task, setup-test] - command: subprocess.exec type: test params: binary: "bash" include_expansions_in_env: [KMS_TLS_TESTCASE] - args: ["${RUN_TASK}", evg-test-kms] + args: [*run-task, evg-test-kms] run-kmip-tests: - command: subprocess.exec @@ -585,21 +586,21 @@ functions: GO_BUILD_TAGS: cse include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, MONGO_GO_DRIVER_COMPRESSOR] - args: ["${RUN_TASK}", setup-test] + args: [*run-task, setup-test] - command: subprocess.exec type: test params: binary: "bash" env: KMS_MOCK_SERVERS_RUNNING: "true" - args: ["${RUN_TASK}", evg-test-kmip] + args: [*run-task, evg-test-kmip] run-fuzz-tests: - command: subprocess.exec type: test params: binary: "bash" - args: ["${RUN_TASK}", run-fuzz] + args: [*run-task, run-fuzz] pre: - func: setup-system @@ -1558,7 +1559,7 @@ tasks: type: test params: binary: "bash" - args: ["${RUN_TASK}", test-awskms] + args: [*run-task, test-awskms] - name: "testawskms-fail-task" # testawskms-fail-task runs without environment variables. @@ -1570,7 +1571,7 @@ tasks: binary: "bash" env: EXPECT_ERROR: 'status=400' - args: ["${RUN_TASK}", test-awskms] + args: [*run-task, test-awskms] - name: "testazurekms-task" commands: @@ -1639,7 +1640,7 @@ tasks: LAMBDA_STACK_NAME: dbx-go-lambda AWS_REGION: us-east-1 include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - args: ["${RUN_TASK}", evg-test-deployed-lambda-aws] + args: [*run-task, evg-test-deployed-lambda-aws] - name: "oidc-auth-test" commands: From 793769e22525023f13cec2bbdea0cfd8c9096690 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 5 Oct 2024 11:27:42 -0500 Subject: [PATCH 21/27] try top level aliases --- .evergreen/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index fa2dae30f8..a9132a1c35 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -25,13 +25,17 @@ timeout: binary: bash args: [ls, -la] +aliases: + working-dir: &working-dir src/go.mongodb.org/mongo-driver + run-task: &run-task src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh + functions: setup-system: # Executes clone and applies the submitted patch, if any - command: git.get_project type: system params: - directory: src/go.mongodb.org/mongo-driver + directory: *working-dir # Make an env.sh and evergreen expansion file with dynamic values - command: subprocess.exec params: @@ -41,7 +45,6 @@ functions: GOROOT: ${GO_DIST} IS_PATCH: ${is_patch} VERSION_ID: ${version_id} - RUN_TASK: &run-task src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh args: - ".evergreen/setup-system.sh" - command: expansions.update From c8f152d7ef5290529acf9e974b02255b9844dc73 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 5 Oct 2024 16:48:15 -0500 Subject: [PATCH 22/27] clean up alias --- .evergreen/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a9132a1c35..707a657d7a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -25,17 +25,13 @@ timeout: binary: bash args: [ls, -la] -aliases: - working-dir: &working-dir src/go.mongodb.org/mongo-driver - run-task: &run-task src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh - functions: setup-system: # Executes clone and applies the submitted patch, if any - command: git.get_project type: system params: - directory: *working-dir + directory: src/go.mongodb.org/mongo-driver # Make an env.sh and evergreen expansion file with dynamic values - command: subprocess.exec params: @@ -45,6 +41,8 @@ functions: GOROOT: ${GO_DIST} IS_PATCH: ${is_patch} VERSION_ID: ${version_id} + # Define an alias for the task runner script. + RUN_TASK_ALIAS: &run-task src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh args: - ".evergreen/setup-system.sh" - command: expansions.update From 38aa59f31b647f05a7815fc10773bbecb409ae73 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Oct 2024 05:26:51 -0500 Subject: [PATCH 23/27] update alias --- .evergreen/config.yml | 78 +++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 707a657d7a..e40da78fee 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -42,7 +42,7 @@ functions: IS_PATCH: ${is_patch} VERSION_ID: ${version_id} # Define an alias for the task runner script. - RUN_TASK_ALIAS: &run-task src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh + TASK_RUNNER_ALIAS: &task_runner src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh args: - ".evergreen/setup-system.sh" - command: expansions.update @@ -173,7 +173,7 @@ functions: env: OIDC: oidc include_expansions_in_env: [DRIVERS_TOOLS, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - args: [*run-task, test-oidc] + args: [*task-runner, test-oidc] run-task: - command: shell.exec @@ -210,7 +210,7 @@ functions: env: BASE_SHA: "${revision}" HEAD_SHA: "${github_commit}" - args: [*run-task, api-report] + args: [*task-runner, api-report] "add PR labels": - command: subprocess.exec @@ -220,7 +220,7 @@ functions: env: COMMIT: "${github_commit}" PR_TASK: apply-labels - args: [*run-task, pr-task] + args: [*task-runner, pr-task] "add PR reviewer": - command: subprocess.exec @@ -230,7 +230,7 @@ functions: env: COMMIT: "${github_commit}" PR_TASK: assign-reviewer - args: [*run-task, pr-task] + args: [*task-runner, pr-task] "backport pr": - command: subprocess.exec @@ -240,7 +240,7 @@ functions: env: COMMIT: "${github_commit}" PR_TASK: backport-pr - args: [*run-task, pr-task] + args: [*task-runner, pr-task] send-perf-data: - command: perf.send @@ -256,12 +256,12 @@ functions: params: binary: bash include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - args: [*run-task, setup-test, --, enterprise-plain] + args: [*task-runner, setup-test, --, enterprise-plain] - command: subprocess.exec type: test params: binary: bash - args: [*run-task, --silent, evg-test-enterprise-auth] + args: [*task-runner, --silent, evg-test-enterprise-auth] run-enterprise-gssapi-auth-tests: - command: ec2.assume_role @@ -272,12 +272,12 @@ functions: params: binary: bash include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - args: [*run-task, setup-test, --, enterprise-gssapi] + args: [*task-runner, setup-test, --, enterprise-gssapi] - command: subprocess.exec type: test params: binary: bash - args: [*run-task, --silent, evg-test-enterprise-auth] + args: [*task-runner, --silent, evg-test-enterprise-auth] run-atlas-test: - command: ec2.assume_role @@ -288,12 +288,12 @@ functions: params: binary: bash include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - args: [*run-task, setup-test, --, atlas-connect] + args: [*task-runner, setup-test, --, atlas-connect] - command: subprocess.exec type: test params: binary: bash - args: [*run-task, test-atlas-connect] + args: [*task-runner, test-atlas-connect] run-ocsp-test: - command: shell.exec @@ -323,12 +323,12 @@ functions: GO_BUILD_TAGS: cse include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, MONGO_GO_DRIVER_COMPRESSOR, REQUIRE_API_VERSION, SKIP_CRYPT_SHARED_LIB, CRYPT_SHARED_LIB_PATH] - args: [*run-task, setup-test] + args: [*task-runner, setup-test] - command: subprocess.exec type: test params: binary: bash - args: [*run-task, evg-test-versioned-api] + args: [*task-runner, evg-test-versioned-api] run-load-balancer-tests: - command: subprocess.exec @@ -336,24 +336,24 @@ functions: params: binary: bash include_expansions_in_env: [SINGLE_MONGOS_LB_URI, MULTI_MONGOS_LB_URI, AUTH, SSL, MONGO_GO_DRIVER_COMPRESSOR] - args: [*run-task, setup-test, --, load-balancer] + args: [*task-runner, setup-test, --, load-balancer] - command: subprocess.exec type: test params: binary: bash - args: [*run-task, evg-test-load-balancers] + args: [*task-runner, evg-test-load-balancers] run-serverless-tests: - command: subprocess.exec type: test params: binary: "bash" - args: [*run-task, setup-test, --, serverless] + args: [*task-runner, setup-test, --, serverless] - command: subprocess.exec type: test params: binary: "bash" - args: [*run-task, evg-test-serverless] + args: [*task-runner, evg-test-serverless] run-atlas-data-lake-test: - command: subprocess.exec @@ -365,26 +365,26 @@ functions: SSL: nossl TOPOLOGY: server MONGODB_URI: "mongodb://mhuser:pencil@localhost" - args: [*run-task, setup-test] + args: [*task-runner, setup-test] - command: subprocess.exec type: test params: binary: "bash" - args: [*run-task, evg-test-atlas-data-lake] + args: [*task-runner, evg-test-atlas-data-lake] run-docker-test: - command: subprocess.exec type: test params: binary: "bash" - args: [*run-task, run-docker, --, test-short] + args: [*task-runner, run-docker, --, test-short] - command: subprocess.exec type: test params: binary: "bash" env: TOPOLOGY: sharded_cluster - args: [*run-task, run-docker, --, test-short] + args: [*task-runner, run-docker, --, test-short] run-valid-ocsp-server: - command: shell.exec @@ -467,7 +467,7 @@ functions: binary: "bash" env: TEST_SEARCH_INDEX: "${MONGODB_URI}" - args: [*run-task, evg-test-search-index] + args: [*task-runner, evg-test-search-index] add-aws-auth-variables-to-file: - command: ec2.assume_role @@ -486,14 +486,14 @@ functions: type: test params: binary: "bash" - args: [*run-task, evg-test-aws, --, regular] + args: [*task-runner, evg-test-aws, --, regular] run-aws-auth-test-with-assume-role-credentials: - command: subprocess.exec type: test params: binary: "bash" - args: [*run-task, evg-test-aws, --, assume-role] + args: [*task-runner, evg-test-aws, --, assume-role] run-aws-auth-test-with-aws-EC2-credentials: - command: subprocess.exec @@ -501,21 +501,21 @@ functions: params: binary: bash include_expansions_in_env: [SKIP_EC2_AUTH_TEST] - args: [*run-task, evg-test-aws, --, ec2] + args: [*task-runner, evg-test-aws, --, ec2] run-aws-auth-test-with-aws-credentials-as-environment-variables: - command: subprocess.exec type: test params: binary: "bash" - args: [*run-task, evg-test-aws, --, env-creds] + args: [*task-runner, evg-test-aws, --, env-creds] run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables: - command: subprocess.exec type: test params: binary: "bash" - args: [*run-task, evg-test-aws, --, session-creds] + args: [*task-runner, evg-test-aws, --, session-creds] run-aws-ECS-auth-test: - command: subprocess.exec @@ -524,7 +524,7 @@ functions: binary: "bash" working_dir: src include_expansions_in_env: [SKIP_ECS_AUTH_TEST] - args: [*run-task, evg-test-aws-ecs] + args: [*task-runner, evg-test-aws-ecs] run-aws-auth-test-with-aws-web-identity-credentials: - command: subprocess.exec @@ -532,7 +532,7 @@ functions: params: binary: bash include_expansions_in_env: [SKIP_WEB_IDENTITY_AUTH_TEST] - args: [*run-task, evg-test-aws, --, web-identity] + args: [*task-runner, evg-test-aws, --, web-identity] - command: subprocess.exec type: test params: @@ -540,7 +540,7 @@ functions: env: AWS_ROLE_SESSION_NAME: test include_expansions_in_env: [SKIP_WEB_IDENTITY_AUTH_TEST] - args: [*run-task, evg-test-aws, --, web-identity] + args: [*task-runner, evg-test-aws, --, web-identity] start-cse-servers: - command: ec2.assume_role @@ -570,13 +570,13 @@ functions: GO_BUILD_TAGS: cse include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, MONGO_GO_DRIVER_COMPRESSOR] - args: [*run-task, setup-test] + args: [*task-runner, setup-test] - command: subprocess.exec type: test params: binary: "bash" include_expansions_in_env: [KMS_TLS_TESTCASE] - args: [*run-task, evg-test-kms] + args: [*task-runner, evg-test-kms] run-kmip-tests: - command: subprocess.exec @@ -587,21 +587,21 @@ functions: GO_BUILD_TAGS: cse include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, MONGO_GO_DRIVER_COMPRESSOR] - args: [*run-task, setup-test] + args: [*task-runner, setup-test] - command: subprocess.exec type: test params: binary: "bash" env: KMS_MOCK_SERVERS_RUNNING: "true" - args: [*run-task, evg-test-kmip] + args: [*task-runner, evg-test-kmip] run-fuzz-tests: - command: subprocess.exec type: test params: binary: "bash" - args: [*run-task, run-fuzz] + args: [*task-runner, run-fuzz] pre: - func: setup-system @@ -1560,7 +1560,7 @@ tasks: type: test params: binary: "bash" - args: [*run-task, test-awskms] + args: [*task-runner, test-awskms] - name: "testawskms-fail-task" # testawskms-fail-task runs without environment variables. @@ -1572,7 +1572,7 @@ tasks: binary: "bash" env: EXPECT_ERROR: 'status=400' - args: [*run-task, test-awskms] + args: [*task-runner, test-awskms] - name: "testazurekms-task" commands: @@ -1641,7 +1641,7 @@ tasks: LAMBDA_STACK_NAME: dbx-go-lambda AWS_REGION: us-east-1 include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - args: [*run-task, evg-test-deployed-lambda-aws] + args: [*task-runner, evg-test-deployed-lambda-aws] - name: "oidc-auth-test" commands: From 42c6c8458f74e30d17cbf33a5aa8ed8e24c72538 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Oct 2024 12:52:54 -0500 Subject: [PATCH 24/27] fix alias --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e40da78fee..ee262eacae 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -42,7 +42,7 @@ functions: IS_PATCH: ${is_patch} VERSION_ID: ${version_id} # Define an alias for the task runner script. - TASK_RUNNER_ALIAS: &task_runner src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh + TASK_RUNNER_ALIAS: &task-runner src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh args: - ".evergreen/setup-system.sh" - command: expansions.update From a320a383e6061698d2ace36433c1403ab8ae2557 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Oct 2024 13:45:07 -0500 Subject: [PATCH 25/27] fix handling of go path --- etc/golangci-lint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/golangci-lint.sh b/etc/golangci-lint.sh index 8c72fe75df..ddc716e62c 100755 --- a/etc/golangci-lint.sh +++ b/etc/golangci-lint.sh @@ -10,7 +10,9 @@ export GOARCH= # Keep this in sync with go version used in static-analysis Evergreen build variant. go install golang.org/dl/go1.22.7@latest go1.22.7 download -PATH="$(go1.22.7 env GOROOT)/bin:$PATH" +GOPATH="$(go1.22.7 env GOROOT)" +PATH="$GOPATH/bin:$PATH" +export GOPATH export PATH go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.1 From c0232445d8624cc30a19a17554714ada625915f0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Oct 2024 13:50:01 -0500 Subject: [PATCH 26/27] try goroot --- etc/golangci-lint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/golangci-lint.sh b/etc/golangci-lint.sh index ddc716e62c..544e3940dd 100755 --- a/etc/golangci-lint.sh +++ b/etc/golangci-lint.sh @@ -10,9 +10,9 @@ export GOARCH= # Keep this in sync with go version used in static-analysis Evergreen build variant. go install golang.org/dl/go1.22.7@latest go1.22.7 download -GOPATH="$(go1.22.7 env GOROOT)" +GOROOT="$(go1.22.7 env GOROOT)" PATH="$GOPATH/bin:$PATH" -export GOPATH +export GOROOT export PATH go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.1 From 881426264d4afd7a417d7ca8a427aa96284dc7c4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Oct 2024 13:56:20 -0500 Subject: [PATCH 27/27] try goroot --- etc/golangci-lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/golangci-lint.sh b/etc/golangci-lint.sh index 544e3940dd..eada915a43 100755 --- a/etc/golangci-lint.sh +++ b/etc/golangci-lint.sh @@ -11,7 +11,7 @@ export GOARCH= go install golang.org/dl/go1.22.7@latest go1.22.7 download GOROOT="$(go1.22.7 env GOROOT)" -PATH="$GOPATH/bin:$PATH" +PATH="$GOROOT/bin:$PATH" export GOROOT export PATH go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.1