diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 46c86103ad..753c1ac28b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -42,3 +42,23 @@ post: - func: "upload mo artifacts" - func: "upload test results" - func: "cleanup" + +tasks: + - name: resync_specs + commands: + - command: subprocess.exec + params: + binary: bash + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + args: + - .evergreen/scripts/resync-all-specs.sh + working_dir: src + +buildvariants: + - name: resync_specs + display_name: "Resync Specs" + run_on: rhel80-small + cron: '0 16 * * MON' + patchable: false + tasks: + - name: resync_specs diff --git a/.evergreen/patch/PYTHON-4884.patch b/.evergreen/patch/PYTHON-4884.patch new file mode 100644 index 0000000000..0ef66e072a --- /dev/null +++ b/.evergreen/patch/PYTHON-4884.patch @@ -0,0 +1,12 @@ +diff --git a/test/bson_corpus/datetime.json b/test/bson_corpus/datetime.json +index f857afdc..1554341d 100644 +--- a/test/bson_corpus/datetime.json ++++ b/test/bson_corpus/datetime.json +@@ -24,6 +24,7 @@ + { + "description" : "Y10K", + "canonical_bson" : "1000000009610000DC1FD277E6000000", ++ "relaxed_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}", + "canonical_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}" + }, + { diff --git a/.evergreen/patch/PYTHON-4918.patch b/.evergreen/patch/PYTHON-4918.patch new file mode 100644 index 0000000000..5f409c5870 --- /dev/null +++ b/.evergreen/patch/PYTHON-4918.patch @@ -0,0 +1,24 @@ +diff --git a/test/connection_monitoring/pool-create-min-size-error.json b/test/connection_monitoring/pool-create-min-size-error.json +index 1c744b85..509b2a23 100644 +--- a/test/connection_monitoring/pool-create-min-size-error.json ++++ b/test/connection_monitoring/pool-create-min-size-error.json +@@ -49,15 +49,15 @@ + "type": "ConnectionCreated", + "address": 42 + }, ++ { ++ "type": "ConnectionPoolCleared", ++ "address": 42 ++ }, + { + "type": "ConnectionClosed", + "address": 42, + "connectionId": 42, + "reason": "error" +- }, +- { +- "type": "ConnectionPoolCleared", +- "address": 42 + } + ], + "ignore": [ diff --git a/.evergreen/patch/PYTHON-4931.patch b/.evergreen/patch/PYTHON-4931.patch new file mode 100644 index 0000000000..ad7086b378 --- /dev/null +++ b/.evergreen/patch/PYTHON-4931.patch @@ -0,0 +1,93 @@ +diff --git a/test/gridfs/delete.json b/test/gridfs/delete.json +index 277b9ed7..9a9b22fc 100644 +--- a/test/gridfs/delete.json ++++ b/test/gridfs/delete.json +@@ -497,7 +497,7 @@ + } + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ], +@@ -650,7 +650,7 @@ + } + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ], +diff --git a/test/gridfs/download.json b/test/gridfs/download.json +index f0cb8517..67658ac5 100644 +--- a/test/gridfs/download.json ++++ b/test/gridfs/download.json +@@ -338,7 +338,7 @@ + } + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ] +@@ -370,7 +370,7 @@ + } + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ] +@@ -402,7 +402,7 @@ + } + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ] +@@ -471,7 +471,7 @@ + } + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ] +@@ -514,7 +514,7 @@ + } + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ] +diff --git a/test/gridfs/downloadByName.json b/test/gridfs/downloadByName.json +index 7b20933c..45abaf7b 100644 +--- a/test/gridfs/downloadByName.json ++++ b/test/gridfs/downloadByName.json +@@ -290,7 +290,7 @@ + "filename": "xyz" + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ] +@@ -306,7 +306,7 @@ + "revision": 999 + }, + "expectError": { +- "isError": true ++ "isClientError": true + } + } + ] diff --git a/.evergreen/patch/PYTHON-5237.patch b/.evergreen/patch/PYTHON-5237.patch new file mode 100644 index 0000000000..01de56b6c8 --- /dev/null +++ b/.evergreen/patch/PYTHON-5237.patch @@ -0,0 +1,48 @@ +diff --git a/test/sessions/driver-sessions-dirty-session-errors.json b/test/sessions/driver-sessions-dirty-session-errors.json +index 6aa1da1d..d7a1c6ab 100644 +--- a/test/sessions/driver-sessions-dirty-session-errors.json ++++ b/test/sessions/driver-sessions-dirty-session-errors.json +@@ -347,7 +347,9 @@ + "x": 1 + } + }, +- "new": false, ++ "new": { ++ "$$unsetOrMatches": false ++ }, + "lsid": { + "$$sessionLsid": "session0" + }, +@@ -375,7 +377,9 @@ + "x": 1 + } + }, +- "new": false, ++ "new": { ++ "$$unsetOrMatches": false ++ }, + "lsid": { + "$$sessionLsid": "session0" + }, +@@ -627,7 +631,9 @@ + "x": 1 + } + }, +- "new": false, ++ "new": { ++ "$$unsetOrMatches": false ++ }, + "lsid": { + "$$type": "object" + }, +@@ -655,7 +661,9 @@ + "x": 1 + } + }, +- "new": false, ++ "new": { ++ "$$unsetOrMatches": false ++ }, + "lsid": { + "$$type": "object" + }, diff --git a/.evergreen/remove-unimplemented-tests.sh b/.evergreen/remove-unimplemented-tests.sh new file mode 100755 index 0000000000..8c5414891e --- /dev/null +++ b/.evergreen/remove-unimplemented-tests.sh @@ -0,0 +1,46 @@ +#!/bin/bash +PYMONGO=$(dirname "$(cd "$(dirname "$0")" || exit; pwd)") + +rm -rf $PYMONGO/test/server_selection/logging +rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894 +rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873 +rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json # PYTHON-5143 +rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json # PYTHON-5143 +rm $PYMONGO/test/client-side-encryption/spec/unified/localSchema.json # PYTHON-5143 +rm $PYMONGO/test/client-side-encryption/spec/unified/maxWireVersion.json # PYTHON-5143 +rm $PYMONGO/test/unified-test-format/valid-pass/poc-queryable-encryption.json # PYTHON-5143 +rm $PYMONGO/test/gridfs/rename.json # PYTHON-4931 +rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-application-error.json # PYTHON-4918 +rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-checkout-error.json # PYTHON-4918 +rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-min-pool-size-error.json # PYTHON-4918 + +# Python doesn't implement DRIVERS-3064 +rm $PYMONGO/test/collection_management/listCollections-rawdata.json +rm $PYMONGO/test/crud/unified/aggregate-rawdata.json +rm $PYMONGO/test/crud/unified/bulkWrite-deleteMany-rawdata.json +rm $PYMONGO/test/crud/unified/bulkWrite-deleteOne-rawdata.json +rm $PYMONGO/test/crud/unified/bulkWrite-replaceOne-rawdata.json +rm $PYMONGO/test/crud/unified/bulkWrite-updateMany-rawdata.json +rm $PYMONGO/test/crud/unified/bulkWrite-updateOne-rawdata.json +rm $PYMONGO/test/crud/unified/client-bulkWrite-delete-rawdata.json +rm $PYMONGO/test/crud/unified/client-bulkWrite-replaceOne-rawdata.json +rm $PYMONGO/test/crud/unified/client-bulkWrite-update-rawdata.json +rm $PYMONGO/test/crud/unified/count-rawdata.json +rm $PYMONGO/test/crud/unified/countDocuments-rawdata.json +rm $PYMONGO/test/crud/unified/db-aggregate-rawdata.json +rm $PYMONGO/test/crud/unified/deleteMany-rawdata.json +rm $PYMONGO/test/crud/unified/deleteOne-rawdata.json +rm $PYMONGO/test/crud/unified/distinct-rawdata.json +rm $PYMONGO/test/crud/unified/estimatedDocumentCount-rawdata.json +rm $PYMONGO/test/crud/unified/find-rawdata.json +rm $PYMONGO/test/crud/unified/findOneAndDelete-rawdata.json +rm $PYMONGO/test/crud/unified/findOneAndReplace-rawdata.json +rm $PYMONGO/test/crud/unified/findOneAndUpdate-rawdata.json +rm $PYMONGO/test/crud/unified/insertMany-rawdata.json +rm $PYMONGO/test/crud/unified/insertOne-rawdata.json +rm $PYMONGO/test/crud/unified/replaceOne-rawdata.json +rm $PYMONGO/test/crud/unified/updateMany-rawdata.json +rm $PYMONGO/test/crud/unified/updateOne-rawdata.json +rm $PYMONGO/test/index_management/index-rawdata.json + +echo "done removing unimplemented tests" diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index d7dfafbba9..d567b380a6 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -45,9 +45,12 @@ then fi # Ensure the JSON files are up to date. -cd $SPECS/source -make -cd - +if ! [ -n "${CI:-}" ] +then + cd $SPECS/source + make + cd - +fi # cpjson unified-test-format/tests/invalid unified-test-format/invalid # * param1: Path to spec tests dir in specifications repo # * param2: Path to where the corresponding tests live in Python. @@ -110,7 +113,6 @@ do cmap|CMAP|connection-monitoring-and-pooling) cpjson connection-monitoring-and-pooling/tests/logging connection_logging cpjson connection-monitoring-and-pooling/tests/cmap-format connection_monitoring - rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873 ;; apm|APM|command-monitoring|command_monitoring) cpjson command-logging-and-monitoring/tests/monitoring command_monitoring @@ -174,7 +176,6 @@ do ;; server-selection|server_selection) cpjson server-selection/tests/ server_selection - rm -rf $PYMONGO/test/server_selection/logging cpjson server-selection/tests/logging server_selection_logging ;; server-selection-logging|server_selection_logging) @@ -186,7 +187,6 @@ do transactions|transactions-convenient-api) cpjson transactions/tests/ transactions cpjson transactions-convenient-api/tests/ transactions-convenient-api - rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894 ;; unified|unified-test-format) cpjson unified-test-format/tests/ unified-test-format/ diff --git a/.evergreen/scripts/create-pr.sh b/.evergreen/scripts/create-pr.sh new file mode 100755 index 0000000000..a5e49bb211 --- /dev/null +++ b/.evergreen/scripts/create-pr.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +tools="$(realpath -s "../drivers-tools")" +pushd $tools/.evergreen/github_app || exit + +owner="mongodb" +repo="mongo-python-driver" + +# Bootstrap the app. +echo "bootstrapping" +source utils.sh +bootstrap drivers/comment-bot + +# Run the app. +source ./secrets-export.sh + +# Get a github access token for the git checkout. +echo "Getting github token..." + +token=$(bash ./get-access-token.sh $repo $owner) +if [ -z "${token}" ]; then + echo "Failed to get github access token!" + popd || exit + exit 1 +fi +echo "Getting github token... done." +popd || exit + +# Make the git checkout and create a new branch. +echo "Creating the git checkout..." +branch="spec-resync-"$(date '+%m-%d-%Y') + +git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git +git checkout -b $branch "origin/master" +git add ./test +git commit -am "resyncing specs $(date '+%m-%d-%Y')" +echo "Creating the git checkout... done." + +git push origin $branch +resp=$(curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"master\"}" \ + --url https://api.github.com/repos/$owner/$repo/pulls) +echo $resp | jq '.html_url' +echo "Creating the PR... done." + +rm -rf $tools diff --git a/.evergreen/scripts/resync-all-specs.py b/.evergreen/scripts/resync-all-specs.py new file mode 100644 index 0000000000..347e51b310 --- /dev/null +++ b/.evergreen/scripts/resync-all-specs.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import argparse +import os +import pathlib +import subprocess +from subprocess import CalledProcessError + + +def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None: + """Actually sync the specs""" + for spec in os.scandir(directory): + if not spec.is_dir(): + continue + + if spec.name in ["asynchronous"]: + continue + try: + subprocess.run( + ["bash", "./.evergreen/resync-specs.sh", spec.name], # noqa: S603, S607 + capture_output=True, + text=True, + check=True, + ) + except CalledProcessError as exc: + errored[spec.name] = exc.stderr + + +def apply_patches(): + subprocess.run(["bash", "./.evergreen/remove-unimplemented-tests.sh"], check=True) # noqa: S603, S607 + subprocess.run(["git apply -R --allow-empty ./.evergreen/patch/*"], shell=True, check=True) # noqa: S602, S607 + + +def check_new_spec_directories(directory: pathlib.Path) -> list[str]: + """Check to see if there are any directories in the spec repo that don't exist in pymongo/test""" + spec_dir = pathlib.Path(os.environ["MDB_SPECS"]) / "source" + spec_set = { + entry.name.replace("-", "_") + for entry in os.scandir(spec_dir) + if entry.is_dir() + and (pathlib.Path(entry.path) / "tests").is_dir() + and len(list(os.scandir(pathlib.Path(entry.path) / "tests"))) > 1 + } + test_set = {entry.name.replace("-", "_") for entry in os.scandir(directory) if entry.is_dir()} + known_mappings = { + "ocsp_support": "ocsp", + "client_side_operations_timeout": "csot", + "mongodb_handshake": "handshake", + "load_balancers": "load_balancer", + "atlas_data_lake_testing": "atlas", + "connection_monitoring_and_pooling": "connection_monitoring", + "command_logging_and_monitoring": "command_logging", + "initial_dns_seedlist_discovery": "srv_seedlist", + "server_discovery_and_monitoring": "sdam_monitoring", + } + + for k, v in known_mappings.items(): + if k in spec_set: + spec_set.remove(k) + spec_set.add(v) + return list(spec_set - test_set) + + +def write_summary(errored: dict[str, str], new: list[str]) -> None: + """Generate the PR description""" + pr_body = "" + process = subprocess.run( + ["git diff --name-only | awk -F'/' '{print $2}' | sort | uniq"], # noqa: S607 + shell=True, # noqa: S602 + capture_output=True, + text=True, + check=True, + ) + succeeded = [i for i in process.stdout.strip().split() if "data/mci/" not in i] + if len(succeeded) > 0: + pr_body += "The following specs were changed:\n -" + pr_body += "\n -".join(succeeded) + pr_body += "\n" + if len(errored) > 0: + pr_body += "\n\nThe following spec syncs encountered errors:\n -" + for k, v in errored.items(): + pr_body += f"\n -{k}\n```{v}\n```" + pr_body += "\n" + if len(new) > 0: + pr_body += "\n\nThe following directories are in the specification repository and not in our test directory:\n -" + pr_body += "\n -".join(new) + pr_body += "\n" + if pr_body != "": + with open("spec_sync.txt", "w") as f: + # replacements made for proper json + f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t")) + + +def main(): + directory = pathlib.Path("./test") + errored: dict[str, str] = {} + resync_specs(directory, errored) + apply_patches() + new = check_new_spec_directories(directory) + write_summary(errored, new) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Python Script to resync all specs and generate summary for PR." + ) + parser.add_argument("filename", help="Name of file for the summary to be written into.") + args = parser.parse_args() + main() diff --git a/.evergreen/scripts/resync-all-specs.sh b/.evergreen/scripts/resync-all-specs.sh new file mode 100755 index 0000000000..f0464ae6f2 --- /dev/null +++ b/.evergreen/scripts/resync-all-specs.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Run spec syncing script and create PR + +# SETUP +SRC_URL="https://github.com/mongodb/specifications.git" +# needs to be set for resunc-specs.sh +SPEC_SRC="$(realpath -s "../specifications")" +SCRIPT="$(realpath -s "./.evergreen/resync-specs.sh")" + +# Clone the spec repo if the directory does not exist +if [[ ! -d $SPEC_SRC ]]; then + git clone $SRC_URL $SPEC_SRC + if [[ $? -ne 0 ]]; then + echo "Error: Failed to clone repository." + exit 1 + fi +fi + +# Set environment variable to the cloned spec repo for resync-specs.sh +export MDB_SPECS="$SPEC_SRC" + +# Check that resync-specs.sh exists and is executable +if [[ ! -x $SCRIPT ]]; then + echo "Error: $SCRIPT not found or is not executable." + exit 1 +fi + +PR_DESC="spec_sync.txt" + +# run python script that actually does all the resyncing +/opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py "$PR_DESC" + + +if [[ -f $PR_DESC ]]; then + # changes were made -> call scrypt to create PR for us + .evergreen/scripts/create-pr.sh "$PR_DESC" + rm "$PR_DESC" +fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca98584602..994815b9cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -432,6 +432,40 @@ update in PyMongo. This is primarily helpful if you are implementing a new feature in PyMongo that has spec tests already implemented, or if you are attempting to validate new spec tests in PyMongo. +### Automated Specification Test Resyncing +There is a script (`/.evergreen/scripts/resync-all-specs.sh`) that will +automatically run once a week to resync all the specs with the [specifications +repo](https://github.com/mongodb/specifications). +If there are changes, a PR will be generated by mongodb-drivers-pr-bot. +If any errors occurred, the PR description will display the name of the spec along +with stderr from the `bash resync-spec.sh ` command. + +Naturally, there will be spec tests that will fail if merged since the associated fix +or behavioral change has not been implemented yet. For existing tests that have been +modified, we will add these changes to a patch file (in `/.evergreen/patch`). Each patch +file will be named after a PYTHON ticket and the contents of the file will be the associated +test differences between PyMongo's tests and the specification. +To be clear, changes listed in these patch files will be *undone* by the script, and thus won't +be added to pymongo. + +In the case that a new test is added to the spec repo (i.e., a whole new file or folder in the spec repo), +and the associated work isn't implemented yet, we will add the test's path to +`.evergreen/remove-unimplemented-tests.sh` along with a comment indicating the associated PYTHON ticket. + +These patch files and/or unimplemented paths must be removed as part of the PR that implements +the fix or behavioral change. + +#### Adding to a patch file +Assuming the changes are committed somewhere, to add to any of the +patch files, run `git diff` to show the desired changes and paste the +results into the patch file. + +For example, there are new test files on the most recent commit of the current branch that are failing. +To add those changes to `new.patch`, I would do the following: +```bash +git diff HEAD~1 path/to/new/file >> .evergreen/patch/new.patch +``` + ## Making a Release Follow the [Python Driver Release Process Wiki](https://wiki.corp.mongodb.com/display/DRIVERS/Python+Driver+Release+Process).