From 69bb5d6d22a13bacab2aaf570d30ac7298d479be Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Mon, 11 Mar 2024 13:09:26 -0400 Subject: [PATCH 1/4] Convert legacy data lake tests to unified JAVA-5354 --- .evergreen/run-atlas-data-lake-test.sh | 2 +- .../test/resources/atlas-data-lake/README.rst | 66 ------------- .../resources/atlas-data-lake/aggregate.json | 53 ----------- .../estimatedDocumentCount.json | 27 ------ .../resources/atlas-data-lake/getMore.json | 58 ----------- .../atlas-data-lake/listCollections.json | 25 ----- .../atlas-data-lake/listDatabases.json | 24 ----- .../resources/atlas-data-lake/runCommand.json | 31 ------ .../atlas-data-lake-testing/aggregate.json | 84 ++++++++++++++++ .../estimatedDocumentCount.json | 56 +++++++++++ .../atlas-data-lake-testing}/find.json | 49 ++++++++-- .../atlas-data-lake-testing/getMore.json | 95 +++++++++++++++++++ .../listCollections.json | 48 ++++++++++ .../listDatabases.json | 41 ++++++++ .../atlas-data-lake-testing/runCommand.json | 54 +++++++++++ .../mongodb/client/AbstractUnifiedTest.java | 17 ++-- .../com/mongodb/client/AtlasDataLakeTest.java | 66 ------------- .../unified/UnifiedAtlasDataLakeTest.java | 45 +++++++++ .../client/unified/UnifiedCrudHelper.java | 60 ++++++------ 19 files changed, 501 insertions(+), 400 deletions(-) delete mode 100644 driver-core/src/test/resources/atlas-data-lake/README.rst delete mode 100644 driver-core/src/test/resources/atlas-data-lake/aggregate.json delete mode 100644 driver-core/src/test/resources/atlas-data-lake/estimatedDocumentCount.json delete mode 100644 driver-core/src/test/resources/atlas-data-lake/getMore.json delete mode 100644 driver-core/src/test/resources/atlas-data-lake/listCollections.json delete mode 100644 driver-core/src/test/resources/atlas-data-lake/listDatabases.json delete mode 100644 driver-core/src/test/resources/atlas-data-lake/runCommand.json create mode 100644 driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/aggregate.json create mode 100644 driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/estimatedDocumentCount.json rename driver-core/src/test/resources/{atlas-data-lake => unified-test-format/atlas-data-lake-testing}/find.json (52%) create mode 100644 driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/getMore.json create mode 100644 driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/listCollections.json create mode 100644 driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/listDatabases.json create mode 100644 driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/runCommand.json delete mode 100644 driver-sync/src/test/functional/com/mongodb/client/AtlasDataLakeTest.java create mode 100644 driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedAtlasDataLakeTest.java diff --git a/.evergreen/run-atlas-data-lake-test.sh b/.evergreen/run-atlas-data-lake-test.sh index 3be9191c0c6..07938018d6e 100755 --- a/.evergreen/run-atlas-data-lake-test.sh +++ b/.evergreen/run-atlas-data-lake-test.sh @@ -23,4 +23,4 @@ DATA_LAKE_URI="mongodb://mhuser:pencil@localhost" echo "Running Atlas Data Lake tests with Java ${JAVA_VERSION}" ./gradlew -version ./gradlew -PjavaVersion=${JAVA_VERSION} -Dorg.mongodb.test.data.lake=true -Dorg.mongodb.test.uri=${DATA_LAKE_URI} \ - --info driver-sync:test --tests AtlasDataLake*Test + --info driver-sync:test --tests *AtlasDataLake*Test diff --git a/driver-core/src/test/resources/atlas-data-lake/README.rst b/driver-core/src/test/resources/atlas-data-lake/README.rst deleted file mode 100644 index 0957dab73f6..00000000000 --- a/driver-core/src/test/resources/atlas-data-lake/README.rst +++ /dev/null @@ -1,66 +0,0 @@ -===================== -Atlas Data Lake Tests -===================== - -.. contents:: - ----- - -Introduction -============ - -The YAML and JSON files in this directory tree are platform-independent tests -that drivers can use to assert compatibility with `Atlas Data Lake `_. - -Running these integration tests will require a running ``mongohoused`` -with data available in its ``test.driverdata`` collection. See the -`ADL directory in drivers-evergreen-tools `_ -and `10gen/mongohouse README `_ -for more information. - -Several prose tests, which are not easily expressed in YAML, are also presented -in this file. Those tests will need to be manually implemented by each driver. - -Test Format -=========== - -The same as the `CRUD Spec Test format <../../crud/tests/README.rst#Test-Format>`_. - -Prose Tests -=========== - -The following tests MUST be implemented to fully test compatibility with -Atlas Data Lake. - -#. Test that the driver properly constructs and issues a - `killCursors `_ - command to Atlas Data Lake. For this test, configure an APM listener on a - client and execute a query that will leave a cursor open on the server (e.g. - specify ``batchSize=2`` for a query that would match 3+ documents). Drivers - MAY iterate the cursor if necessary to execute the initial ``find`` command - but MUST NOT iterate further to avoid executing a ``getMore``. - - Observe the CommandSucceededEvent event for the ``find`` command and extract - the cursor's ID and namespace from the response document's ``cursor.id`` and - ``cursor.ns`` fields, respectively. Destroy the cursor object and observe - a CommandStartedEvent and CommandSucceededEvent for the ``killCursors`` - command. Assert that the cursor ID and target namespace in the outgoing - command match the values from the ``find`` command's CommandSucceededEvent. - Assert that the ``killCursors`` CommandSucceededEvent indicates that the - expected cursor was killed in the ``cursorsKilled`` field. - - Note: this test assumes that drivers only issue a ``killCursors`` command - internally when destroying a cursor that may still exist on the server. If - a driver constructs and issues ``killCursors`` commands in other ways (e.g. - public API), this test MUST be adapted to test all such code paths. - -#. Test that the driver can establish a connection with Atlas Data Lake - without authentication. For these tests, create a MongoClient using a - valid connection string without auth credentials and execute a ping - command. - -#. Test that the driver can establish a connection with Atlas Data Lake - with authentication. For these tests, create a MongoClient using a - valid connection string with SCRAM-SHA-1 and credentials from the - drivers-evergreen-tools ADL configuration and execute a ping command. - Repeat this test using SCRAM-SHA-256. diff --git a/driver-core/src/test/resources/atlas-data-lake/aggregate.json b/driver-core/src/test/resources/atlas-data-lake/aggregate.json deleted file mode 100644 index 99995bca415..00000000000 --- a/driver-core/src/test/resources/atlas-data-lake/aggregate.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "collection_name": "driverdata", - "database_name": "test", - "tests": [ - { - "description": "Aggregate with pipeline (project, sort, limit)", - "operations": [ - { - "object": "collection", - "name": "aggregate", - "arguments": { - "pipeline": [ - { - "$project": { - "_id": 0 - } - }, - { - "$sort": { - "a": 1 - } - }, - { - "$limit": 2 - } - ] - }, - "result": [ - { - "a": 1, - "b": 2, - "c": 3 - }, - { - "a": 2, - "b": 3, - "c": 4 - } - ] - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "aggregate": "driverdata" - } - } - } - ] - } - ] -} diff --git a/driver-core/src/test/resources/atlas-data-lake/estimatedDocumentCount.json b/driver-core/src/test/resources/atlas-data-lake/estimatedDocumentCount.json deleted file mode 100644 index 997a3ab3fcd..00000000000 --- a/driver-core/src/test/resources/atlas-data-lake/estimatedDocumentCount.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "collection_name": "driverdata", - "database_name": "test", - "tests": [ - { - "description": "estimatedDocumentCount succeeds", - "operations": [ - { - "object": "collection", - "name": "estimatedDocumentCount", - "result": 15 - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "count": "driverdata" - }, - "command_name": "count", - "database_name": "test" - } - } - ] - } - ] -} diff --git a/driver-core/src/test/resources/atlas-data-lake/getMore.json b/driver-core/src/test/resources/atlas-data-lake/getMore.json deleted file mode 100644 index 9aa2c2de1d2..00000000000 --- a/driver-core/src/test/resources/atlas-data-lake/getMore.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "collection_name": "driverdata", - "database_name": "test", - "tests": [ - { - "description": "A successful find event with getMore", - "comment": "UPDATED final batchSize to 3 as batchSize is no longer calculated see: DRIVERS-1448 ", - "operations": [ - { - "object": "collection", - "name": "find", - "arguments": { - "filter": { - "a": { - "$gte": 2 - } - }, - "sort": { - "a": 1 - }, - "batchSize": 3, - "limit": 4 - } - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "find": "driverdata", - "filter": { - "a": { - "$gte": 2 - } - }, - "sort": { - "a": 1 - }, - "batchSize": 3, - "limit": 4 - }, - "command_name": "find", - "database_name": "test" - } - }, - { - "command_started_event": { - "command": { - "batchSize": 3 - }, - "command_name": "getMore", - "database_name": "cursors" - } - } - ] - } - ] -} diff --git a/driver-core/src/test/resources/atlas-data-lake/listCollections.json b/driver-core/src/test/resources/atlas-data-lake/listCollections.json deleted file mode 100644 index 8d8a8f6c1b6..00000000000 --- a/driver-core/src/test/resources/atlas-data-lake/listCollections.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "database_name": "test", - "tests": [ - { - "description": "ListCollections succeeds", - "operations": [ - { - "name": "listCollections", - "object": "database" - } - ], - "expectations": [ - { - "command_started_event": { - "command_name": "listCollections", - "database_name": "test", - "command": { - "listCollections": 1 - } - } - } - ] - } - ] -} \ No newline at end of file diff --git a/driver-core/src/test/resources/atlas-data-lake/listDatabases.json b/driver-core/src/test/resources/atlas-data-lake/listDatabases.json deleted file mode 100644 index f8ec9a0bf41..00000000000 --- a/driver-core/src/test/resources/atlas-data-lake/listDatabases.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "tests": [ - { - "description": "ListDatabases succeeds", - "operations": [ - { - "name": "listDatabases", - "object": "client" - } - ], - "expectations": [ - { - "command_started_event": { - "command_name": "listDatabases", - "database_name": "admin", - "command": { - "listDatabases": 1 - } - } - } - ] - } - ] -} \ No newline at end of file diff --git a/driver-core/src/test/resources/atlas-data-lake/runCommand.json b/driver-core/src/test/resources/atlas-data-lake/runCommand.json deleted file mode 100644 index f72e863ba5d..00000000000 --- a/driver-core/src/test/resources/atlas-data-lake/runCommand.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "database_name": "test", - "tests": [ - { - "description": "ping succeeds using runCommand", - "operations": [ - { - "name": "runCommand", - "object": "database", - "command_name": "ping", - "arguments": { - "command": { - "ping": 1 - } - } - } - ], - "expectations": [ - { - "command_started_event": { - "command_name": "ping", - "database_name": "test", - "command": { - "ping": 1 - } - } - } - ] - } - ] -} \ No newline at end of file diff --git a/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/aggregate.json b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/aggregate.json new file mode 100644 index 00000000000..68a3467c71c --- /dev/null +++ b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/aggregate.json @@ -0,0 +1,84 @@ +{ + "description": "aggregate", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "driverdata" + } + } + ], + "tests": [ + { + "description": "Aggregate with pipeline (project, sort, limit)", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$project": { + "_id": 0 + } + }, + { + "$sort": { + "a": 1 + } + }, + { + "$limit": 2 + } + ] + }, + "expectResult": [ + { + "a": 1, + "b": 2, + "c": 3 + }, + { + "a": 2, + "b": 3, + "c": 4 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "driverdata" + }, + "commandName": "aggregate", + "databaseName": "test" + } + } + ] + } + ] + } + ] +} diff --git a/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/estimatedDocumentCount.json b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/estimatedDocumentCount.json new file mode 100644 index 00000000000..b7515a44182 --- /dev/null +++ b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/estimatedDocumentCount.json @@ -0,0 +1,56 @@ +{ + "description": "estimatedDocumentCount", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "driverdata" + } + } + ], + "tests": [ + { + "description": "estimatedDocumentCount succeeds", + "operations": [ + { + "object": "collection0", + "name": "estimatedDocumentCount", + "expectResult": 15 + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "count": "driverdata" + }, + "commandName": "count", + "databaseName": "test" + } + } + ] + } + ] + } + ] +} diff --git a/driver-core/src/test/resources/atlas-data-lake/find.json b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/find.json similarity index 52% rename from driver-core/src/test/resources/atlas-data-lake/find.json rename to driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/find.json index 8a3468a135e..d0652dc720f 100644 --- a/driver-core/src/test/resources/atlas-data-lake/find.json +++ b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/find.json @@ -1,12 +1,36 @@ { - "collection_name": "driverdata", - "database_name": "test", + "description": "find", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "driverdata" + } + } + ], "tests": [ { "description": "Find with projection and sort", "operations": [ { - "object": "collection", + "object": "collection0", "name": "find", "arguments": { "filter": { @@ -22,7 +46,7 @@ }, "limit": 5 }, - "result": [ + "expectResult": [ { "a": 5, "b": 6, @@ -51,13 +75,20 @@ ] } ], - "expectations": [ + "expectEvents": [ { - "command_started_event": { - "command": { - "find": "driverdata" + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "driverdata" + }, + "commandName": "find", + "databaseName": "test" + } } - } + ] } ] } diff --git a/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/getMore.json b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/getMore.json new file mode 100644 index 00000000000..e7c600d5258 --- /dev/null +++ b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/getMore.json @@ -0,0 +1,95 @@ +{ + "description": "getMore", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "driverdata" + } + } + ], + "tests": [ + { + "description": "A successful find event with getMore", + "operations": [ + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "a": { + "$gte": 2 + } + }, + "sort": { + "a": 1 + }, + "batchSize": 3, + "limit": 4 + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "driverdata", + "filter": { + "a": { + "$gte": 2 + } + }, + "sort": { + "a": 1 + }, + "batchSize": 3, + "limit": 4 + }, + "commandName": "find", + "databaseName": "test" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": { + "$$type": "string" + }, + "batchSize": 3 + }, + "commandName": "getMore", + "databaseName": "cursors" + } + } + ] + } + ] + } + ] +} diff --git a/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/listCollections.json b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/listCollections.json new file mode 100644 index 00000000000..642e7ed328a --- /dev/null +++ b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/listCollections.json @@ -0,0 +1,48 @@ +{ + "description": "listCollections", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + } + ], + "tests": [ + { + "description": "ListCollections succeeds", + "operations": [ + { + "object": "database0", + "name": "listCollections" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1 + }, + "commandName": "listCollections", + "databaseName": "test" + } + } + ] + } + ] + } + ] +} diff --git a/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/listDatabases.json b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/listDatabases.json new file mode 100644 index 00000000000..64506ee54e4 --- /dev/null +++ b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/listDatabases.json @@ -0,0 +1,41 @@ +{ + "description": "listDatabases", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + } + ], + "tests": [ + { + "description": "ListCollections succeeds", + "operations": [ + { + "object": "client0", + "name": "listDatabases" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listDatabases": 1 + }, + "commandName": "listDatabases", + "databaseName": "admin" + } + } + ] + } + ] + } + ] +} diff --git a/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/runCommand.json b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/runCommand.json new file mode 100644 index 00000000000..325b6b3f30a --- /dev/null +++ b/driver-core/src/test/resources/unified-test-format/atlas-data-lake-testing/runCommand.json @@ -0,0 +1,54 @@ +{ + "description": "runCommand", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + } + ], + "tests": [ + { + "description": "ping succeeds using runCommand", + "operations": [ + { + "object": "database0", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "ping": 1 + }, + "commandName": "ping", + "databaseName": "test" + } + } + ] + } + ] + } + ] +} diff --git a/driver-sync/src/test/functional/com/mongodb/client/AbstractUnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/AbstractUnifiedTest.java index 6cba714f065..f4e00343891 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/AbstractUnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/AbstractUnifiedTest.java @@ -68,7 +68,6 @@ import static com.mongodb.ClusterFixture.getConnectionString; import static com.mongodb.ClusterFixture.getMultiMongosConnectionString; -import static com.mongodb.ClusterFixture.isDataLakeTest; import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionAtLeast; import static com.mongodb.ClusterFixture.serverVersionLessThan; @@ -170,16 +169,14 @@ public void setUp() { killAllSessions(); - if (!isDataLakeTest()) { - try { + try { + collectionHelper.create(collectionName, new CreateCollectionOptions(), WriteConcern.MAJORITY); + } catch (MongoCommandException e) { + // Older sharded clusters sometimes reply with this error. Work around it by retrying once. + if (e.getErrorCode() == 11601 && isSharded() && serverVersionLessThan(4, 2)) { collectionHelper.create(collectionName, new CreateCollectionOptions(), WriteConcern.MAJORITY); - } catch (MongoCommandException e) { - // Older sharded clusters sometimes reply with this error. Work around it by retrying once. - if (e.getErrorCode() == 11601 && isSharded() && serverVersionLessThan(4, 2)) { - collectionHelper.create(collectionName, new CreateCollectionOptions(), WriteConcern.MAJORITY); - } else { - throw e; - } + } else { + throw e; } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/AtlasDataLakeTest.java b/driver-sync/src/test/functional/com/mongodb/client/AtlasDataLakeTest.java deleted file mode 100644 index 672eae4208f..00000000000 --- a/driver-sync/src/test/functional/com/mongodb/client/AtlasDataLakeTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2008-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.mongodb.client; - -import com.mongodb.MongoClientSettings; -import org.bson.BsonArray; -import org.bson.BsonDocument; -import org.bson.BsonString; -import org.bson.BsonValue; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import util.JsonPoweredTestHelper; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static com.mongodb.ClusterFixture.isDataLakeTest; -import static com.mongodb.JsonTestServerVersionChecker.skipTest; -import static com.mongodb.client.Fixture.getDefaultDatabaseName; - -// See https://github.com/mongodb/specifications/tree/master/source/transactions/tests -@RunWith(Parameterized.class) -public class AtlasDataLakeTest extends AbstractUnifiedTest { - public AtlasDataLakeTest(final String filename, final String description, final String databaseName, final String collectionName, - final BsonArray data, final BsonDocument definition, final boolean skipTest) { - super(filename, description, databaseName, collectionName, data, definition, skipTest || !isDataLakeTest(), true); - } - - @Parameterized.Parameters(name = "{0}: {1}") - public static Collection data() throws URISyntaxException, IOException { - List data = new ArrayList<>(); - for (File file : JsonPoweredTestHelper.getTestFiles("/atlas-data-lake")) { - BsonDocument testDocument = JsonPoweredTestHelper.getTestDocument(file); - for (BsonValue test : testDocument.getArray("tests")) { - data.add(new Object[]{file.getName(), test.asDocument().getString("description").getValue(), - testDocument.getString("database_name", new BsonString(getDefaultDatabaseName())).getValue(), - testDocument.getString("collection_name", new BsonString("test")).getValue(), - testDocument.getArray("data", new BsonArray()), test.asDocument(), skipTest(testDocument, test.asDocument())}); - } - } - return data; - } - - @Override - protected MongoClient createMongoClient(final MongoClientSettings settings) { - return MongoClients.create(settings); - } -} diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedAtlasDataLakeTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedAtlasDataLakeTest.java new file mode 100644 index 00000000000..9bc43e5f25d --- /dev/null +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedAtlasDataLakeTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.client.unified; + +import com.mongodb.lang.Nullable; +import org.bson.BsonArray; +import org.bson.BsonDocument; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collection; + +import static com.mongodb.ClusterFixture.isDataLakeTest; +import static org.junit.Assume.assumeTrue; + +public class UnifiedAtlasDataLakeTest extends UnifiedSyncTest { + + public UnifiedAtlasDataLakeTest(@SuppressWarnings("unused") final String fileDescription, + @SuppressWarnings("unused") final String testDescription, final String schemaVersion, + @Nullable final BsonArray runOnRequirements, final BsonArray entities, final BsonArray initialData, + final BsonDocument definition) { + super(schemaVersion, runOnRequirements, entities, initialData, definition); + assumeTrue(isDataLakeTest()); + } + + @Parameterized.Parameters(name = "{0}: {1}") + public static Collection data() throws URISyntaxException, IOException { + return getTestData("unified-test-format/atlas-data-lake-testing"); + } +} diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedCrudHelper.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedCrudHelper.java index 992b8a80f01..0316f146d78 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedCrudHelper.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedCrudHelper.java @@ -267,7 +267,7 @@ OperationResult executeListDatabaseNames(final BsonDocument operation) { OperationResult executeListCollections(final BsonDocument operation) { MongoDatabase database = entities.getDatabase(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); ListCollectionsIterable iterable = session == null ? database.listCollections(BsonDocument.class) @@ -294,7 +294,7 @@ OperationResult executeListCollections(final BsonDocument operation) { OperationResult executeListCollectionNames(final BsonDocument operation) { MongoDatabase database = entities.getDatabase(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); MongoIterable iterable = session == null ? database.listCollectionNames() @@ -379,7 +379,7 @@ OperationResult createFindCursor(final BsonDocument operation) { @NonNull private FindIterable createFindIterable(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); BsonDocument filter = arguments.getDocument("filter", new BsonDocument()); FindIterable iterable = session == null ? collection.find(filter) : collection.find(session, filter); @@ -446,7 +446,7 @@ private FindIterable createFindIterable(final BsonDocument operati OperationResult executeDistinct(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); BsonString fieldName = arguments.getString("fieldName"); @@ -479,7 +479,7 @@ OperationResult executeDistinct(final BsonDocument operation) { OperationResult executeFindOneAndUpdate(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonDocument filter = arguments.getDocument("filter").asDocument(); BsonValue update = arguments.get("update"); @@ -555,7 +555,7 @@ OperationResult executeFindOneAndUpdate(final BsonDocument operation) { OperationResult executeFindOneAndReplace(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); BsonDocument filter = arguments.getDocument("filter").asDocument(); BsonDocument replacement = arguments.getDocument("replacement").asDocument(); @@ -620,7 +620,7 @@ OperationResult executeFindOneAndReplace(final BsonDocument operation) { OperationResult executeFindOneAndDelete(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); BsonDocument filter = arguments.getDocument("filter").asDocument(); FindOneAndDeleteOptions options = new FindOneAndDeleteOptions(); @@ -669,7 +669,7 @@ OperationResult executeFindOneAndDelete(final BsonDocument operation) { OperationResult executeAggregate(final BsonDocument operation) { String entityName = operation.getString("object").getValue(); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); List pipeline = arguments.getArray("pipeline").stream().map(BsonValue::asDocument).collect(toList()); AggregateIterable iterable; @@ -726,7 +726,7 @@ OperationResult executeAggregate(final BsonDocument operation) { OperationResult executeDeleteOne(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonDocument filter = arguments.getDocument("filter"); ClientSession session = getSession(arguments); DeleteOptions options = getDeleteOptions(arguments); @@ -742,7 +742,7 @@ OperationResult executeDeleteOne(final BsonDocument operation) { OperationResult executeDeleteMany(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonDocument filter = arguments.getDocument("filter"); ClientSession session = getSession(arguments); DeleteOptions options = getDeleteOptions(arguments); @@ -766,7 +766,7 @@ private BsonDocument toExpected(final DeleteResult result) { OperationResult executeUpdateOne(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); BsonDocument filter = arguments.getDocument("filter"); BsonValue update = arguments.get("update"); @@ -790,7 +790,7 @@ OperationResult executeUpdateOne(final BsonDocument operation) { OperationResult executeUpdateMany(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonDocument filter = arguments.getDocument("filter"); BsonValue update = arguments.get("update"); ClientSession session = getSession(arguments); @@ -813,7 +813,7 @@ OperationResult executeUpdateMany(final BsonDocument operation) { OperationResult executeReplaceOne(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); BsonDocument filter = arguments.getDocument("filter"); BsonDocument replacement = arguments.getDocument("replacement"); @@ -846,7 +846,7 @@ private BsonDocument toExpected(final UpdateResult result) { OperationResult executeInsertOne(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); BsonDocument document = arguments.getDocument("document").asDocument(); InsertOneOptions options = new InsertOneOptions(); @@ -880,7 +880,7 @@ private BsonDocument toExpected(final InsertOneResult result) { OperationResult executeInsertMany(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); List documents = arguments.getArray("documents").stream().map(BsonValue::asDocument).collect(toList()); ClientSession session = getSession(arguments); InsertManyOptions options = new InsertManyOptions(); @@ -921,7 +921,7 @@ private BsonDocument toExpected(final InsertManyResult result) { OperationResult executeBulkWrite(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); List> requests = arguments.getArray("requests").stream() .map(value -> toWriteModel(value.asDocument())).collect(toList()); @@ -1166,9 +1166,9 @@ OperationResult executeAbortTransaction(final BsonDocument operation) { OperationResult executeWithTransaction(final BsonDocument operation, final OperationAsserter operationAsserter) { ClientSession session = entities.getSession(operation.getString("object").getValue()); - BsonArray callback = operation.getDocument("arguments").getArray("callback"); + BsonArray callback = operation.getDocument("arguments", new BsonDocument()).getArray("callback"); TransactionOptions.Builder optionsBuilder = TransactionOptions.builder(); - for (Map.Entry entry : operation.getDocument("arguments").entrySet()) { + for (Map.Entry entry : operation.getDocument("arguments", new BsonDocument()).entrySet()) { switch (entry.getKey()) { case "callback": break; @@ -1201,10 +1201,10 @@ OperationResult executeWithTransaction(final BsonDocument operation, final Opera public OperationResult executeDropCollection(final BsonDocument operation) { MongoDatabase database = entities.getDatabase(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); String collectionName = arguments.getString("collection").getValue(); - if (operation.getDocument("arguments").size() > 1) { + if (operation.getDocument("arguments", new BsonDocument()).size() > 1) { throw new UnsupportedOperationException("Unexpected arguments"); } @@ -1216,7 +1216,7 @@ public OperationResult executeDropCollection(final BsonDocument operation) { public OperationResult executeCreateCollection(final BsonDocument operation) { MongoDatabase database = entities.getDatabase(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); String collectionName = arguments.getString("collection").getValue(); ClientSession session = getSession(arguments); @@ -1286,7 +1286,7 @@ public OperationResult executeCreateCollection(final BsonDocument operation) { public OperationResult executeModifyCollection(final BsonDocument operation) { MongoDatabase database = entities.getDatabase(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); String collectionName = arguments.getString("collection").getValue(); ClientSession session = getSession(arguments); @@ -1319,7 +1319,7 @@ public OperationResult executeModifyCollection(final BsonDocument operation) { public OperationResult executeRenameCollection(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); String newCollectionName = arguments.getString("to").getValue(); ClientSession session = getSession(arguments); RenameCollectionOptions options = new RenameCollectionOptions(); @@ -1417,7 +1417,7 @@ private TimeSeriesGranularity createTimeSeriesGranularity(final String value) { OperationResult executeCreateSearchIndex(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonDocument model = arguments.getDocument("model"); BsonDocument definition = model.getDocument("definition"); @@ -1434,7 +1434,7 @@ OperationResult executeCreateSearchIndex(final BsonDocument operation) { OperationResult executeCreateSearchIndexes(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonArray models = arguments.getArray("models"); List searchIndexModels = models.stream() @@ -1449,7 +1449,7 @@ OperationResult executeCreateSearchIndexes(final BsonDocument operation) { OperationResult executeUpdateSearchIndex(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonDocument definition = arguments.getDocument("definition"); String name = arguments.getString("name").getValue(); @@ -1461,7 +1461,7 @@ OperationResult executeUpdateSearchIndex(final BsonDocument operation) { OperationResult executeDropSearchIndex(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); String name = arguments.getString("name").getValue(); return resultOf(() -> { @@ -1524,7 +1524,7 @@ private ListSearchIndexesIterable createListSearchIndexesIterable( public OperationResult executeCreateIndex(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonDocument keys = arguments.getDocument("keys").asDocument(); ClientSession session = getSession(arguments); IndexOptions options = new IndexOptions(); @@ -1557,7 +1557,7 @@ public OperationResult executeCreateIndex(final BsonDocument operation) { public OperationResult executeDropIndex(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); String indexName = arguments.get("name").asString().getValue(); for (Map.Entry cur : arguments.entrySet()) { @@ -1687,7 +1687,7 @@ public OperationResult executeRunCommand(final BsonDocument operation) { public OperationResult executeCountDocuments(final BsonDocument operation) { MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); - BsonDocument arguments = operation.getDocument("arguments"); + BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); BsonDocument filter = arguments.getDocument("filter"); ClientSession session = getSession(arguments); CountOptions options = new CountOptions(); From 016b48ec74bcd0910cabc42f10eea3907b18a551 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Mon, 11 Mar 2024 14:00:50 -0400 Subject: [PATCH 2/4] Require filter for find --- .../com/mongodb/client/unified/UnifiedCrudHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedCrudHelper.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedCrudHelper.java index 0316f146d78..f228812271e 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedCrudHelper.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedCrudHelper.java @@ -381,7 +381,7 @@ private FindIterable createFindIterable(final BsonDocument operati MongoCollection collection = entities.getCollection(operation.getString("object").getValue()); BsonDocument arguments = operation.getDocument("arguments", new BsonDocument()); ClientSession session = getSession(arguments); - BsonDocument filter = arguments.getDocument("filter", new BsonDocument()); + BsonDocument filter = arguments.getDocument("filter"); FindIterable iterable = session == null ? collection.find(filter) : collection.find(session, filter); for (Map.Entry cur : arguments.entrySet()) { switch (cur.getKey()) { From e4792f20e79428b1dc932492cdf08845ed6501b8 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 14 Mar 2024 08:42:10 -0400 Subject: [PATCH 3/4] Revert change to AbstractUnifiedTest since the whole file will be removed soon, and this will just create a merge conflict. --- .../com/mongodb/client/AbstractUnifiedTest.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/AbstractUnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/AbstractUnifiedTest.java index f4e00343891..6cba714f065 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/AbstractUnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/AbstractUnifiedTest.java @@ -68,6 +68,7 @@ import static com.mongodb.ClusterFixture.getConnectionString; import static com.mongodb.ClusterFixture.getMultiMongosConnectionString; +import static com.mongodb.ClusterFixture.isDataLakeTest; import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionAtLeast; import static com.mongodb.ClusterFixture.serverVersionLessThan; @@ -169,14 +170,16 @@ public void setUp() { killAllSessions(); - try { - collectionHelper.create(collectionName, new CreateCollectionOptions(), WriteConcern.MAJORITY); - } catch (MongoCommandException e) { - // Older sharded clusters sometimes reply with this error. Work around it by retrying once. - if (e.getErrorCode() == 11601 && isSharded() && serverVersionLessThan(4, 2)) { + if (!isDataLakeTest()) { + try { collectionHelper.create(collectionName, new CreateCollectionOptions(), WriteConcern.MAJORITY); - } else { - throw e; + } catch (MongoCommandException e) { + // Older sharded clusters sometimes reply with this error. Work around it by retrying once. + if (e.getErrorCode() == 11601 && isSharded() && serverVersionLessThan(4, 2)) { + collectionHelper.create(collectionName, new CreateCollectionOptions(), WriteConcern.MAJORITY); + } else { + throw e; + } } } From 80788a1b1d9799532e5d887aa4c5a49c5381d9a7 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Tue, 19 Mar 2024 12:31:17 -0400 Subject: [PATCH 4/4] Don't killAllSessions on Data Lake --- .../functional/com/mongodb/client/unified/UnifiedTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index a7cb9793c4b..76ee46d4e3b 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -72,6 +72,7 @@ import java.util.stream.Collectors; import static com.mongodb.ClusterFixture.getServerVersion; +import static com.mongodb.ClusterFixture.isDataLakeTest; import static com.mongodb.client.Fixture.getMongoClient; import static com.mongodb.client.Fixture.getMongoClientSettings; import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime; @@ -223,7 +224,9 @@ public void setUp() { throw new AssumptionViolatedException(definition.getString("skipReason").getValue()); } - killAllSessions(); + if (!isDataLakeTest()) { + killAllSessions(); + } startingClusterTime = addInitialDataAndGetClusterTime();