diff --git a/source/client-side-operations-timeout/tests/runCursorCommand.json b/source/client-side-operations-timeout/tests/runCursorCommand.json new file mode 100644 index 0000000000..5fc0be3399 --- /dev/null +++ b/source/client-side-operations-timeout/tests/runCursorCommand.json @@ -0,0 +1,583 @@ +{ + "description": "runCursorCommand", + "schemaVersion": "1.9", + "runOnRequirements": [ + { + "minServerVersion": "4.4" + } + ], + "createEntities": [ + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "client": { + "id": "commandClient", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ] + } + }, + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ], + "ignoreCommandMonitoringEvents": [ + "killCursors" + ] + } + }, + { + "database": { + "id": "commandDb", + "client": "commandClient", + "databaseName": "commandDb" + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "db" + } + }, + { + "collection": { + "id": "collection", + "database": "db", + "collectionName": "collection" + } + } + ], + "initialData": [ + { + "collectionName": "collection", + "databaseName": "db", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "tests": [ + { + "description": "errors if timeoutMode is set without timeoutMS", + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "command": { + "find": "collection" + }, + "timeoutMode": "cursorLifetime" + }, + "expectError": { + "isClientError": true + } + } + ] + }, + { + "description": "error if timeoutMode is cursorLifetime and cursorType is tailableAwait", + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "command": { + "find": "collection" + }, + "timeoutMode": "cursorLifetime", + "cursorType": "tailableAwait" + }, + "expectError": { + "isClientError": true + } + } + ] + }, + { + "description": "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "find", + "getMore" + ], + "blockConnection": true, + "blockTimeMS": 60 + } + } + } + }, + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "timeoutMS": 100, + "command": { + "find": "collection", + "batchSize": 2 + } + }, + "expectError": { + "isTimeoutError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find", + "command": { + "find": "collection", + "maxTimeMS": { + "$$type": [ + "int", + "long" + ] + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "getMore", + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "maxTimeMS": { + "$$exists": true + } + } + } + } + ] + } + ] + }, + { + "description": "Non=tailable cursor iteration timeoutMS is refreshed for getMore if timeoutMode is iteration - failure", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "getMore" + ], + "blockConnection": true, + "blockTimeMS": 60 + } + } + } + }, + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "command": { + "find": "collection", + "batchSize": 2 + }, + "timeoutMode": "iteration", + "timeoutMS": 100, + "batchSize": 2 + }, + "expectError": { + "isTimeoutError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find", + "databaseName": "db", + "command": { + "find": "collection", + "maxTimeMS": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "getMore", + "databaseName": "db", + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "maxTimeMS": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "Tailable cursor iteration timeoutMS is refreshed for getMore - failure", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "getMore" + ], + "blockConnection": true, + "blockTimeMS": 60 + } + } + } + }, + { + "name": "dropCollection", + "object": "db", + "arguments": { + "collection": "cappedCollection" + } + }, + { + "name": "createCollection", + "object": "db", + "arguments": { + "collection": "cappedCollection", + "capped": true, + "size": 4096, + "max": 3 + }, + "saveResultAsEntity": "cappedCollection" + }, + { + "name": "insertMany", + "object": "cappedCollection", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + }, + { + "name": "createCommandCursor", + "object": "db", + "arguments": { + "commandName": "find", + "command": { + "find": "cappedCollection", + "batchSize": 1, + "tailable": true + }, + "timeoutMode": "iteration", + "timeoutMS": 100, + "batchSize": 1, + "cursorType": "tailable" + }, + "saveResultAsEntity": "tailableCursor" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "tailableCursor" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "tailableCursor", + "expectError": { + "isTimeoutError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "drop" + } + }, + { + "commandStartedEvent": { + "commandName": "create" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "find", + "databaseName": "db", + "command": { + "find": "cappedCollection", + "tailable": true, + "awaitData": { + "$$exists": false + }, + "maxTimeMS": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "getMore", + "databaseName": "db", + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "cappedCollection", + "maxTimeMS": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "Tailable cursor awaitData iteration timeoutMS is refreshed for getMore - failure", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "getMore" + ], + "blockConnection": true, + "blockTimeMS": 60 + } + } + } + }, + { + "name": "dropCollection", + "object": "db", + "arguments": { + "collection": "cappedCollection" + } + }, + { + "name": "createCollection", + "object": "db", + "arguments": { + "collection": "cappedCollection", + "capped": true, + "size": 4096, + "max": 3 + }, + "saveResultAsEntity": "cappedCollection" + }, + { + "name": "insertMany", + "object": "cappedCollection", + "arguments": { + "documents": [ + { + "foo": "bar" + }, + { + "fizz": "buzz" + } + ] + } + }, + { + "name": "createCommandCursor", + "object": "db", + "arguments": { + "command": { + "find": "cappedCollection", + "tailable": true, + "awaitData": true + }, + "cursorType": "tailableAwait", + "batchSize": 1 + }, + "saveResultAsEntity": "tailableCursor" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "tailableCursor" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "tailableCursor", + "expectError": { + "isTimeoutError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "drop" + } + }, + { + "commandStartedEvent": { + "commandName": "create" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "find", + "databaseName": "db", + "command": { + "find": "cappedCollection", + "tailable": true, + "awaitData": true, + "maxTimeMS": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "getMore", + "databaseName": "db", + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "cappedCollection" + } + } + } + ] + } + ] + } + ] +} diff --git a/source/client-side-operations-timeout/tests/runCursorCommand.yml b/source/client-side-operations-timeout/tests/runCursorCommand.yml new file mode 100644 index 0000000000..16a648e028 --- /dev/null +++ b/source/client-side-operations-timeout/tests/runCursorCommand.yml @@ -0,0 +1,304 @@ +description: runCursorCommand + +schemaVersion: '1.9' + +runOnRequirements: + - minServerVersion: "4.4" + +createEntities: + - client: + id: &failPointClient failPointClient + useMultipleMongoses: false + - client: + id: &commandClient commandClient + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent] + - client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent] + ignoreCommandMonitoringEvents: [killCursors] + - database: # For tests that need success event assertions + id: &commandDb commandDb + client: *commandClient + databaseName: *commandDb + - database: + id: &db db + client: *client + databaseName: *db + - collection: + id: &collection collection + database: *db + collectionName: *collection + +initialData: + - collectionName: *collection + databaseName: *db + documents: &documents + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + - { _id: 5, x: 55 } + +tests: + - description: errors if timeoutMode is set without timeoutMS + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: find + command: { find: *collection } + timeoutMode: cursorLifetime + expectError: + isClientError: true + + - description: error if timeoutMode is cursorLifetime and cursorType is tailableAwait + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: find + command: { find: *collection } + timeoutMode: cursorLifetime + cursorType: tailableAwait + expectError: + isClientError: true + + # If timeoutMode is unset, it should default to CURSOR_LIFETIME and the time remaining after the find succeeds should be applied to the getMore + - description: Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset + runOnRequirements: + - serverless: forbid + operations: + # Block find/getMore for 15ms. + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [find, getMore] + blockConnection: true + blockTimeMS: 60 + # Run a find with timeoutMS less than double our failPoint blockTimeMS and + # batchSize less than the total document count will cause a find and a getMore to be sent. + # Both will block for 60ms so together they will go over the timeout. + - name: runCursorCommand + object: *db + arguments: + commandName: find + timeoutMS: 100 + command: { find: *collection, batchSize: 2 } + expectError: + isTimeoutError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + command: + find: *collection + maxTimeMS: { $$type: [int, long] } + - commandStartedEvent: + commandName: getMore + command: + getMore: { $$type: [int, long] } + collection: *collection + maxTimeMS: { $$exists: true } + + # If timeoutMode=ITERATION, timeoutMS applies separately to the initial find and the getMore on the cursor. Neither + # command should have a maxTimeMS field. This is a failure test. The "find" inherits timeoutMS=100 and "getMore" + # commands are blocked for 60ms, causing iteration to fail with a timeout error. + - description: Non=tailable cursor iteration timeoutMS is refreshed for getMore if timeoutMode is iteration - failure + runOnRequirements: + - serverless: forbid + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["getMore"] + blockConnection: true + blockTimeMS: 60 + - name: runCursorCommand + object: *db + arguments: + commandName: find + command: { find: *collection, batchSize: 2 } + timeoutMode: iteration + timeoutMS: 100 + batchSize: 2 + expectError: + isTimeoutError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *db + command: + find: *collection + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: getMore + databaseName: *db + command: + getMore: { $$type: ["int", "long"] } + collection: *collection + maxTimeMS: { $$exists: false } + + # The timeoutMS option should apply separately to the initial "find" and each getMore. This is a failure test. The + # find inherits timeoutMS=100 from the collection and the getMore command blocks for 60ms, causing iteration to fail + # with a timeout error. + - description: Tailable cursor iteration timeoutMS is refreshed for getMore - failure + runOnRequirements: + - serverless: forbid + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["getMore"] + blockConnection: true + blockTimeMS: 60 + - name: dropCollection + object: *db + arguments: + collection: &cappedCollection cappedCollection + - name: createCollection + object: *db + arguments: + collection: *cappedCollection + capped: true + size: 4096 + max: 3 + saveResultAsEntity: *cappedCollection + - name: insertMany + object: *cappedCollection + arguments: + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - name: createCommandCursor + object: *db + arguments: + commandName: find + command: { find: *cappedCollection, batchSize: 1, tailable: true } + timeoutMode: iteration + timeoutMS: 100 + batchSize: 1 + cursorType: tailable + saveResultAsEntity: &tailableCursor tailableCursor + # Iterate the cursor twice: the first iteration will return the document from the batch in the find and the + # second will do a getMore. + - name: iterateUntilDocumentOrError + object: *tailableCursor + - name: iterateUntilDocumentOrError + object: *tailableCursor + expectError: + isTimeoutError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: drop + - commandStartedEvent: + commandName: create + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: find + databaseName: *db + command: + find: *cappedCollection + tailable: true + awaitData: { $$exists: false } + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: getMore + databaseName: *db + command: + getMore: { $$type: ["int", "long"] } + collection: *cappedCollection + maxTimeMS: { $$exists: false } + + # The timeoutMS value should be refreshed for getMore's. This is a failure test. The find inherits timeoutMS=10 from + # the collection and the getMore blocks for 15ms, causing iteration to fail with a timeout error. + - description: Tailable cursor awaitData iteration timeoutMS is refreshed for getMore - failure + runOnRequirements: + - serverless: forbid + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["getMore"] + blockConnection: true + blockTimeMS: 60 + - name: dropCollection + object: *db + arguments: + collection: &cappedCollection cappedCollection + - name: createCollection + object: *db + arguments: + collection: *cappedCollection + capped: true + size: 4096 + max: 3 + saveResultAsEntity: *cappedCollection + - name: insertMany + object: *cappedCollection + arguments: + documents: [ { foo: bar }, { fizz: buzz } ] + - name: createCommandCursor + object: *db + arguments: + command: { find: *cappedCollection, tailable: true, awaitData: true } + cursorType: tailableAwait + batchSize: 1 + saveResultAsEntity: &tailableCursor tailableCursor + # Iterate twice to force a getMore. + - name: iterateUntilDocumentOrError + object: *tailableCursor + - name: iterateUntilDocumentOrError + object: *tailableCursor + expectError: + isTimeoutError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: drop + - commandStartedEvent: + commandName: create + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: find + databaseName: *db + command: + find: *cappedCollection + tailable: true + awaitData: true + maxTimeMS: { $$exists: true } + - commandStartedEvent: + commandName: getMore + databaseName: *db + command: + getMore: { $$type: ["int", "long"] } + collection: *cappedCollection diff --git a/source/find_getmore_killcursors_commands.rst b/source/find_getmore_killcursors_commands.rst index 0f522c9ccf..7dbdde13f3 100644 --- a/source/find_getmore_killcursors_commands.rst +++ b/source/find_getmore_killcursors_commands.rst @@ -263,17 +263,39 @@ The driver **SHOULD NOT attempt to emulate the behavior seen in 3.0 or earlier** Tailable cursors ^^^^^^^^^^^^^^^^ -Tailable cursors have some fundamental changes compared to the existing **OP_QUERY** implementation. To create a tailable cursor you execute the following command: +By default, most cursors are non-tailable. For example, a typical ``find`` cursor is exhausted when all results for the filter have been returned. +MongoDB also supports creating cursors that "tail" or follow the target namespace for new data. This is done via the `find` command's `tailable` option. +Querying a capped collection is one use case for a tailable cursor. +A tailable cursor can receive ``getMore`` responses with an empty ``nextBatch`` array, which does not indicate that the cursor has been exhausted. -.. code:: javascript - - var b = db.runCommand({ find:"t", tailable: true }); +Additionally, tailable cursors may "await data" from the server, which means that the server will wait to respond to the client's ``getMore`` command until new data is available or some period of time has elapsed, whichever comes first. +A tailable ``find`` cursor can be configured using the ``awaitData`` option. +The amount of time the server will wait for new data is based on the ``maxTimeMS`` field of the ``getMore`` (or server default if unspecified). +If the time does expire an empty batch will be returned. +A ``maxTimeMS`` field cannot be sent if the cursor was not configured with ``awaitData=true``. -To create a tailable cursor with **tailable** and **awaitData**, execute the following command: +To create a tailable ``find`` cursor you execute the following command: -.. code:: javascript +.. _find: https://www.mongodb.com/docs/manual/reference/command/find/ - var b = db.runCommand({ find:"t", tailable: true, awaitData: true }); +.. code:: typescript + + interface FindCommand { + /** The namespace to run the find command on */ + find: string; + /** The filter to control what documents are returned */ + filter: BSONDocument; + /** Informs the server whether to keep the cursor open even when there are no results to satisfy the query */ + tailable?: boolean; + /** + * Informs the server whether to block on a `getMore`'s `maxTimeMS` before returning an empty `nextBatch`. + * This must be set if getMores are to include `maxTimeMS` values. + */ + awaitData?: boolean; + /** Controls the amount of milliseconds the server will allow the operations to run for */ + maxTimeMS?: PositiveIntegerNumber; + // ... Note: additional options unrelated to tailable cursors omitted + } If **maxTimeMS** is not set in FindOptions, the driver SHOULD refrain from setting **maxTimeMS** on the **find** or **getMore** commands issued by the driver and allow the server to use its internal default value for **maxTimeMS**. @@ -294,23 +316,54 @@ when sent to a secondary. The OP_QUERY namespace MUST be the same as for the .. _getMore: https://www.mongodb.com/docs/manual/reference/command/getMore/ -The **batchSize** option of **getMore** command MUST be an int32 larger than 0. If **batchSize** is equal to 0 it must be omitted. If **batchSize** is less than 0 it must be turned into a positive integer using **Math.abs** or equivalent function in your language. +.. code:: typescript + + interface GetMoreCommand { + /** Set to the nonzero cursor id */ + getMore: int64; + /** Set to the namespace returned by the initial command response */ + collection: string; + /** + * User configurable document count for the batch returned for this getMore. + * Only attached to command document if nonzero. + */ + batchSize?: PositiveIntegerNumber; + /** + * User configurable time limit enforced by the server. + */ + maxTimeMS?: PositiveIntegerNumber; + /** + * User configurable comment that can be used to identify the operation in logs. + * This can be any BSON value. + */ + comment?: BSONValue; + } On success, the getMore command will return the following: -.. code:: javascript - - { - "cursor": { - "id": , - "ns": , - "nextBatch": [ - ... - ] - }, - "ok": 1 +.. code:: typescript + + interface GetMoreResponse { + ok: 1; + cursor: { + /** The cursor id, this may be equal to zero indicating the cursor is exhausted or closed */ + id: int64; + /** + * The namespace the cursor is running on. + * This value may be different than the namespace the driver started the cursor on, for example, database level aggregations. + */ + ns: string; + /** + * The next batch of documents. + * This array may be empty, in the case of a tailable cursor, which DOES NOT indicate the cursor is exhausted. + */ + nextBatch: BSONArray; + } + // ... Note: additional non-relevant fields omitted } +The driver's local cursor MUST update its ``id`` and ``ns``, as well as store the ``nextBatch`` from every ``getMore`` response. + killCursors ----------- @@ -463,6 +516,7 @@ More in depth information about passing read preferences to Mongos can be found Changelog ========= +:2023-05-10: Improve tailable cursor description and update the ``getMore`` section code blocks. :2022-10-05: Remove spec front matter and reformat changelog. :2022-02-01: Replace examples/tables for find, getMore, and killCursors with server manual links. diff --git a/source/run-command/run-command.rst b/source/run-command/run-command.rst index 8f3a9e3cfc..eadb27ecc7 100644 --- a/source/run-command/run-command.rst +++ b/source/run-command/run-command.rst @@ -44,6 +44,8 @@ Deviations Please refer to `The CRUD specification's Guidance `_ on how APIs may deviate between languages. +Cursor iterating APIs MAY be offered via language syntax or predefined iterable methods. + -------------- ``runCommand`` -------------- @@ -129,6 +131,10 @@ The logical session ID MUST be included under ``lsid`` in the command sent to th * See Driver Sessions' section on `Sending the session ID to the server on all commands `_ +The command sent to the server MUST gossip the ``$clusterTime`` if cluster time support is detected. + +* See Driver Sessions' section on `Gossipping the cluster time `_ + Transactions """""""""""" @@ -187,8 +193,152 @@ Drivers MUST document the behavior of RunCommand if a ``maxTimeMS`` field is al * See Client Side Operations Timeout's section on `runCommand `_ * See Client Side Operations Timeout's section on `runCommand behavior `_ + +-------------------- +``runCursorCommand`` +-------------------- + +Drivers MAY expose a runCursorCommand API with the following syntax. + +.. code:: typescript + + interface Database { + /** + * Takes an argument representing an arbitrary BSON document and executes it against the server. + */ + runCursorCommand(command: BSONDocument, options: RunCursorCommandOptions): RunCommandCursor; + } + + interface RunCursorCommandOptions extends RunCommandOptions { + /** + * This option is an enum with possible values CURSOR_LIFETIME and ITERATION. + * For operations that create cursors, timeoutMS can either cap the lifetime of the cursor or be applied separately to the original operation and all subsequent calls. + * To support both of these use cases, these operations MUST support a timeoutMode option. + * + * @defaultValue CURSOR_LIFETIME + * + * @see https://github.com/mongodb/specifications/blob/master/source/client-side-operations-timeout/client-side-operations-timeout.rst + */ + timeoutMode?: ITERATION | CURSOR_LIFETIME; + + /** + * See the `cursorType` enum defined in the crud specification. + * @see https://github.com/mongodb/specifications/blob/master/source/crud/crud.rst#read + * + * Identifies the type of cursor this is for client side operations timeout to properly apply timeoutMode settings. + * + * A tailable cursor can receive empty `nextBatch` arrays in `getMore` responses. + * However, subsequent `getMore` operations may return documents if new data has become available. + * + * A tailableAwait cursor is an enhancement where instead of dealing with empty responses the server will block until data becomes available. + * + * @defaultValue NON_TAILABLE + */ + cursorType?: CursorType; + } + + /** + * The following are the configurations a driver MUST provide to control how getMores are constructed. + * How the options are controlled should be idiomatic to the driver's language. + * See Executing ``getMore`` Commands. + */ + interface RunCursorCommandGetMoreOptions { + /** Any positive integer is permitted. */ + batchSize?: int; + /** Any non-negative integer is permitted. */ + maxTimeMS?: int; + comment?: BSONValue; + } + +RunCursorCommand implementation details +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +RunCursorCommand provides a way to access MongoDB server commands that return a cursor directly without requiring a driver to implement a bespoke cursor implementation. +The API is intended to be built upon RunCommand and take a document from a user and apply a number of common driver internal concerns before forwarding the command to a server. +A driver can expect that the result from running this command will return a document with a ``cursor`` field and MUST provide the caller with a language native abstraction to continue iterating the results from the server. +If the response from the server does not include a ``cursor`` field the driver MUST throw an error either before returning from ``runCursorCommand`` or upon first iteration of the cursor. + +High level RunCursorCommand steps: + +* Run the cursor creating command provided by the caller and retain the ClientSession used as well as the server the command was executed on. +* Create a local cursor instance and store the ``firstBatch``, ``ns``, and ``id`` from the response. +* When the current batch has been fully iterated, execute a ``getMore`` using the same server the initial command was executed on. +* Store the ``nextBatch`` from the ``getMore`` response and update the cursor's ``id``. +* Continue to execute ``getMore`` commands as needed when the caller empties local batches until the cursor is exhausted or closed (i.e. ``id`` is zero). + +Driver Sessions +""""""""""""""" + +A driver MUST create an implicit ClientSession if none is provided and it MUST be attached for the duration of the cursor's lifetime. +All ``getMore`` commands constructed for this cursor MUST send the same ``lsid`` used on the initial command. +A cursor is considered exhausted or closed when the server reports its ``id`` as zero. +When the cursor is exhausted the client session MUST be ended and the server session returned to the pool as early as possible rather than waiting for a caller to completely iterate the final batch. + +* See Drivers Sessions' section on `Sessions and Cursors `_ + +Server Selection +"""""""""""""""" + +RunCursorCommand MUST support a ``readPreference`` option that MUST be used to determine server selection. +The selected server MUST be used for subsequent ``getMore`` commands. + +Load Balancers +"""""""""""""" + +When in ``loadBalanced`` mode, a driver MUST pin the connection used to execute the initial operation, and reuse it for subsequent ``getMore`` operations. + +* See Load Balancer's section on `Behaviour With Cursors `_ + +Iterating the Cursor +"""""""""""""""""""" + +Drivers MUST provide an API, typically, a method named ``next()``, that returns one document per invocation. +If the cursor's batch is empty and the cursor id is nonzero, the driver MUST perform a ``getMore`` operation. + +Executing ``getMore`` Commands +"""""""""""""""""""""""""""""" + +The cursor API returned to the caller MUST offer an API to configure ``batchSize``, ``maxTimeMS``, and ``comment`` options that are sent on subsequent ``getMore`` commands. +If it is idiomatic for a driver to allow setting these options in ``RunCursorCommandOptions``, the driver MUST document that the options only pertain to ``getMore`` commands. +A driver MAY permit users to change ``getMore`` field settings at any time during the cursor's lifetime and subsequent ``getMore`` commands MUST be constructed with the changes to those fields. +If that API is offered drivers MUST write tests asserting ``getMore`` commands are constructed with any updated fields. + +* See Find, getMore and killCursors commands' section on `GetMore `_ + +Tailable and TailableAwait +"""""""""""""""""""""""""" + +* **See first:** Find, getMore and killCursors commands's section on `Tailable cursors `_ + +It is the responsibility of the caller to construct their initial command with ``awaitData`` and ``tailable`` flags **as well as** inform RunCursorCommand of the ``cursorType`` that should be constructed. +Requesting a ``cursorType`` that does not align with the fields sent to the server on the initial command SHOULD be documented as undefined behavior. + +Resource Cleanup +"""""""""""""""" + +Drivers MUST provide an explicit mechanism for releasing the cursor resources, typically a ``.close()`` method. +If the cursor id is nonzero a KillCursors operation MUST be attempted, the result of the operation SHOULD be ignored. +The ClientSession associated with the cursor MUST be ended and the ServerSession returned to the pool. + +* See Driver Sessions' section on `When sending a killCursors command `_ +* See Find, getMore and killCursors commands' section on `killCursors `_ + +Client Side Operations Timeout +"""""""""""""""""""""""""""""" + +RunCursorCommand MUST provide an optional ``timeoutMS`` option to support client side operations timeout. +Drivers MUST NOT attempt to check the command document for the presence of a ``maxTimeMS`` field. +Drivers MUST document the behavior of RunCursorCommand if a ``maxTimeMS`` field is already set on the command. +Drivers SHOULD raise an error if both ``timeoutMS`` and the ``getMore``-specific ``maxTimeMS`` option are specified (see: `Executing getMore Commands`_). +Drivers MUST document that attempting to set both options can have undefined behavior and is not supported. + +When ``timeoutMS`` and ``timeoutMode`` are provided the driver MUST support timeout functionality as described in the CSOT specification. + +* See Client Side Operations Timeout's section on `Cursors `_ + Changelog ========= +:2023-05-10: Add runCursorCommand API specification. :2023-05-08: ``$readPreference`` is not sent to standalone servers :2023-04-20: Add run command specification. diff --git a/source/run-command/tests/unified/runCommand.json b/source/run-command/tests/unified/runCommand.json index 9c7f912f95..007e514bd7 100644 --- a/source/run-command/tests/unified/runCommand.json +++ b/source/run-command/tests/unified/runCommand.json @@ -123,6 +123,68 @@ } ] }, + { + "description": "always gossips the $clusterTime on the sent command", + "runOnRequirements": [ + { + "topologies": [ + "replicaset", + "sharded" + ] + } + ], + "operations": [ + { + "name": "runCommand", + "object": "db", + "arguments": { + "commandName": "ping", + "command": { + "ping": 1 + } + }, + "expectResult": { + "ok": 1 + } + }, + { + "name": "runCommand", + "object": "db", + "arguments": { + "commandName": "ping", + "command": { + "ping": 1 + } + }, + "expectResult": { + "ok": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "command": { + "ping": 1, + "$clusterTime": { + "$$exists": true + } + }, + "commandName": "ping" + } + } + ] + } + ] + }, { "description": "attaches the provided session lsid to given command", "operations": [ @@ -346,7 +408,7 @@ "insert": "collection", "documents": [ { - "_id": 1 + "foo": "bar" } ], "ordered": true @@ -452,7 +514,7 @@ "insert": "collection", "documents": [ { - "_id": 2 + "foo": "transaction" } ], "ordered": true @@ -480,7 +542,7 @@ "insert": "collection", "documents": [ { - "_id": 2 + "foo": "transaction" } ], "ordered": true, diff --git a/source/run-command/tests/unified/runCommand.yml b/source/run-command/tests/unified/runCommand.yml index edff7f6410..eaa12eff23 100644 --- a/source/run-command/tests/unified/runCommand.yml +++ b/source/run-command/tests/unified/runCommand.yml @@ -67,6 +67,36 @@ tests: $readPreference: { $$exists: false } commandName: ping + - description: always gossips the $clusterTime on the sent command + runOnRequirements: + # Only replicasets and sharded clusters have a $clusterTime + - topologies: [ replicaset, sharded ] + operations: + # We have to run one command to obtain a clusterTime to gossip + - name: runCommand + object: *db + arguments: + commandName: ping + command: { ping: 1 } + expectResult: { ok: 1 } + - name: runCommand + object: *db + arguments: + commandName: ping + command: { ping: 1 } + expectResult: { ok: 1 } + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: ping + # Only check the shape of the second ping which should have the $clusterTime received from the first operation + - commandStartedEvent: + command: + ping: 1 + $clusterTime: { $$exists: true } + commandName: ping + - description: attaches the provided session lsid to given command operations: - name: runCommand @@ -180,7 +210,7 @@ tests: commandName: insert command: insert: *collection - documents: [ { _id: 1 } ] + documents: [ { foo: 'bar' } ] ordered: true expectResult: { ok: 1 } expectEvents: @@ -233,7 +263,7 @@ tests: commandName: insert command: insert: *collection - documents: [ { _id: 2 } ] + documents: [ { foo: 'transaction' } ] ordered: true expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } expectEvents: @@ -242,7 +272,7 @@ tests: - commandStartedEvent: command: insert: *collection - documents: [ { _id: 2 } ] + documents: [ { foo: 'transaction' } ] ordered: true lsid: { $$sessionLsid: *session } txnNumber: 1 diff --git a/source/run-command/tests/unified/runCursorCommand.json b/source/run-command/tests/unified/runCursorCommand.json new file mode 100644 index 0000000000..4f1ec8a01a --- /dev/null +++ b/source/run-command/tests/unified/runCursorCommand.json @@ -0,0 +1,877 @@ +{ + "description": "runCursorCommand", + "schemaVersion": "1.9", + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "connectionReadyEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent" + ] + } + }, + { + "session": { + "id": "session", + "client": "client" + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "db" + } + }, + { + "collection": { + "id": "collection", + "database": "db", + "collectionName": "collection" + } + } + ], + "initialData": [ + { + "collectionName": "collection", + "databaseName": "db", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "tests": [ + { + "description": "successfully executes checkMetadataConsistency cursor creating command", + "runOnRequirements": [ + { + "minServerVersion": "7.0", + "topologies": [ + "sharded" + ] + } + ], + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "checkMetadataConsistency", + "command": { + "checkMetadataConsistency": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "checkMetadataConsistency": 1, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "checkMetadataConsistency" + } + } + ] + } + ] + }, + { + "description": "errors if the command response is not a cursor", + "operations": [ + { + "name": "createCommandCursor", + "object": "db", + "arguments": { + "commandName": "ping", + "command": { + "ping": 1 + } + }, + "expectError": { + "isClientError": true + } + } + ] + }, + { + "description": "creates an implicit session that is reused across getMores", + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "command": { + "find": "collection", + "batchSize": 2 + } + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + }, + { + "name": "assertSameLsidOnLastTwoCommands", + "object": "testRunner", + "arguments": { + "client": "client" + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "collection", + "batchSize": 2, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "accepts an explicit session that is reused across getMores", + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "session": "session", + "command": { + "find": "collection", + "batchSize": 2 + } + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + }, + { + "name": "assertSameLsidOnLastTwoCommands", + "object": "testRunner", + "arguments": { + "client": "client" + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "collection", + "batchSize": 2, + "$db": "db", + "lsid": { + "$$sessionLsid": "session" + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "$db": "db", + "lsid": { + "$$sessionLsid": "session" + } + }, + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "returns pinned connections to the pool when the cursor is exhausted", + "runOnRequirements": [ + { + "topologies": [ + "load-balanced" + ] + } + ], + "operations": [ + { + "name": "createCommandCursor", + "object": "db", + "arguments": { + "commandName": "find", + "batchSize": 2, + "session": "session", + "command": { + "find": "collection", + "batchSize": 2 + } + }, + "saveResultAsEntity": "cursor" + }, + { + "name": "assertNumberConnectionsCheckedOut", + "object": "testRunner", + "arguments": { + "client": "client", + "connections": 1 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor", + "expectResult": { + "_id": 1, + "x": 11 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor", + "expectResult": { + "_id": 2, + "x": 22 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor", + "expectResult": { + "_id": 3, + "x": 33 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor", + "expectResult": { + "_id": 4, + "x": 44 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor", + "expectResult": { + "_id": 5, + "x": 55 + } + }, + { + "name": "assertNumberConnectionsCheckedOut", + "object": "testRunner", + "arguments": { + "client": "client", + "connections": 0 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "collection", + "batchSize": 2, + "$db": "db", + "lsid": { + "$$sessionLsid": "session" + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "$db": "db", + "lsid": { + "$$sessionLsid": "session" + } + }, + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "$db": "db", + "lsid": { + "$$sessionLsid": "session" + } + }, + "commandName": "getMore" + } + } + ] + }, + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionReadyEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "connectionCheckedInEvent": {} + } + ] + } + ] + }, + { + "description": "returns pinned connections to the pool when the cursor is closed", + "runOnRequirements": [ + { + "topologies": [ + "load-balanced" + ] + } + ], + "operations": [ + { + "name": "createCommandCursor", + "object": "db", + "arguments": { + "commandName": "find", + "command": { + "find": "collection", + "batchSize": 2 + } + }, + "saveResultAsEntity": "cursor" + }, + { + "name": "assertNumberConnectionsCheckedOut", + "object": "testRunner", + "arguments": { + "client": "client", + "connections": 1 + } + }, + { + "name": "close", + "object": "cursor" + }, + { + "name": "assertNumberConnectionsCheckedOut", + "object": "testRunner", + "arguments": { + "client": "client", + "connections": 0 + } + } + ] + }, + { + "description": "supports configuring getMore batchSize", + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "batchSize": 5, + "command": { + "find": "collection", + "batchSize": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "collection", + "batchSize": 1, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "batchSize": 5, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "supports configuring getMore maxTimeMS", + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "maxTimeMS": 300, + "command": { + "find": "collection", + "maxTimeMS": 200, + "batchSize": 1 + } + }, + "ignoreResultAndError": true + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "ignoreExtraEvents": true, + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "collection", + "maxTimeMS": 200, + "batchSize": 1, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "$db": "db", + "maxTimeMS": 300, + "lsid": { + "$$exists": true + } + }, + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "supports configuring getMore comment", + "runOnRequirements": [ + { + "minServerVersion": "4.4" + } + ], + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "comment": { + "hello": "getMore" + }, + "command": { + "find": "collection", + "batchSize": 1, + "comment": { + "hello": "find" + } + } + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "collection", + "batchSize": 1, + "comment": { + "hello": "find" + }, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "comment": { + "hello": "getMore" + }, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "does not close the cursor when receiving an empty batch", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "name": "dropCollection", + "object": "db", + "arguments": { + "collection": "cappedCollection" + } + }, + { + "name": "createCollection", + "object": "db", + "arguments": { + "collection": "cappedCollection", + "capped": true, + "size": 4096, + "max": 3 + }, + "saveResultAsEntity": "cappedCollection" + }, + { + "name": "insertMany", + "object": "cappedCollection", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + }, + { + "name": "createCommandCursor", + "object": "db", + "arguments": { + "cursorType": "tailable", + "commandName": "find", + "batchSize": 2, + "command": { + "find": "cappedCollection", + "tailable": true + } + }, + "saveResultAsEntity": "cursor" + }, + { + "name": "iterateOnce", + "object": "cursor" + }, + { + "name": "iterateOnce", + "object": "cursor" + }, + { + "name": "iterateOnce", + "object": "cursor" + }, + { + "name": "close", + "object": "cursor" + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "drop": "cappedCollection" + }, + "commandName": "drop" + } + }, + { + "commandStartedEvent": { + "command": { + "create": "cappedCollection" + }, + "commandName": "create" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "cappedCollection" + }, + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "cappedCollection", + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "cappedCollection", + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "command": { + "killCursors": "cappedCollection", + "cursors": { + "$$type": "array" + } + }, + "commandName": "killCursors" + } + } + ] + } + ] + } + ] +} diff --git a/source/run-command/tests/unified/runCursorCommand.yml b/source/run-command/tests/unified/runCursorCommand.yml new file mode 100644 index 0000000000..1f9bf532c3 --- /dev/null +++ b/source/run-command/tests/unified/runCursorCommand.yml @@ -0,0 +1,391 @@ +description: runCursorCommand + +schemaVersion: '1.9' + +createEntities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, connectionReadyEvent, connectionCheckedOutEvent, connectionCheckedInEvent] + - session: + id: &session session + client: *client + - database: + id: &db db + client: *client + databaseName: *db + - collection: + id: &collection collection + database: *db + collectionName: *collection + +initialData: + - collectionName: collection + databaseName: *db + documents: &documents + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + - { _id: 5, x: 55 } + +tests: + # This is what this API was invented to do. + - description: successfully executes checkMetadataConsistency cursor creating command + runOnRequirements: + - minServerVersion: '7.0' + topologies: [sharded] + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: checkMetadataConsistency + command: { checkMetadataConsistency: 1 } + expectEvents: + - client: *client + eventType: command + events: + - commandStartedEvent: + command: + checkMetadataConsistency: 1 + $db: *db + lsid: { $$exists: true } + commandName: checkMetadataConsistency + + - description: errors if the command response is not a cursor + operations: + - name: createCommandCursor + object: *db + arguments: + commandName: ping + command: { ping: 1 } + expectError: + isClientError: true + + + # Driver Sessions + - description: creates an implicit session that is reused across getMores + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: find + command: { find: *collection, batchSize: 2 } + expectResult: *documents + - name: assertSameLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client + expectEvents: + - client: *client + eventType: command + events: + - commandStartedEvent: + command: + find: *collection + batchSize: 2 + $db: *db + lsid: { $$exists: true } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + $db: *db + lsid: { $$exists: true } + commandName: getMore + + - description: accepts an explicit session that is reused across getMores + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: find + session: *session + command: { find: *collection, batchSize: 2 } + expectResult: *documents + - name: assertSameLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client + expectEvents: + - client: *client + eventType: command + events: + - commandStartedEvent: + command: + find: *collection + batchSize: 2 + $db: *db + lsid: { $$sessionLsid: *session } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + $db: *db + lsid: { $$sessionLsid: *session } + commandName: getMore + + # Load Balancers + - description: returns pinned connections to the pool when the cursor is exhausted + runOnRequirements: + - topologies: [ load-balanced ] + operations: + - name: createCommandCursor + object: *db + arguments: + commandName: find + batchSize: 2 + session: *session + command: { find: *collection, batchSize: 2 } + saveResultAsEntity: &cursor cursor + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client + connections: 1 + - name: iterateUntilDocumentOrError + object: *cursor + expectResult: { _id: 1, x: 11 } + - name: iterateUntilDocumentOrError + object: *cursor + expectResult: { _id: 2, x: 22 } + - name: iterateUntilDocumentOrError + object: *cursor + expectResult: { _id: 3, x: 33 } + - name: iterateUntilDocumentOrError + object: *cursor + expectResult: { _id: 4, x: 44 } + - name: iterateUntilDocumentOrError + object: *cursor + expectResult: { _id: 5, x: 55 } + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client + connections: 0 + expectEvents: + - client: *client + eventType: command + events: + - commandStartedEvent: + command: + find: *collection + batchSize: 2 + $db: *db + lsid: { $$sessionLsid: *session } + commandName: find # 2 documents + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + $db: *db + lsid: { $$sessionLsid: *session } + commandName: getMore # 2 documents + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + $db: *db + lsid: { $$sessionLsid: *session } + commandName: getMore # 1 document + # Total documents: 5 + - client: *client + eventType: cmap + events: + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: returns pinned connections to the pool when the cursor is closed + runOnRequirements: + - topologies: [ load-balanced ] + operations: + - name: createCommandCursor + object: *db + arguments: + commandName: find + command: { find: *collection, batchSize: 2 } + saveResultAsEntity: *cursor + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client + connections: 1 + - name: close + object: *cursor + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client + connections: 0 + + # Iterating the Cursor / Executing GetMores + - description: supports configuring getMore batchSize + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: find + batchSize: 5 + command: { find: *collection, batchSize: 1 } + expectResult: *documents + expectEvents: + - client: *client + eventType: command + events: + - commandStartedEvent: + command: + find: *collection + batchSize: 1 + $db: *db + lsid: { $$exists: true } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + batchSize: 5 + $db: *db + lsid: { $$exists: true } + commandName: getMore + + - description: supports configuring getMore maxTimeMS + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: find + maxTimeMS: 300 + command: { find: *collection, maxTimeMS: 200, batchSize: 1 } + ignoreResultAndError: true + expectEvents: + - client: *client + eventType: command + # The getMore should receive an error here because we do not have the right kind of cursor + # So drivers should run a killCursors, but neither the error nor the killCursors command is relevant to this test + ignoreExtraEvents: true + events: + - commandStartedEvent: + command: + find: *collection + maxTimeMS: 200 + batchSize: 1 + $db: *db + lsid: { $$exists: true } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + $db: *db + maxTimeMS: 300 + lsid: { $$exists: true } + commandName: getMore + + - description: supports configuring getMore comment + runOnRequirements: + - minServerVersion: '4.4' + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: find + comment: { hello: 'getMore' } + command: { find: *collection, batchSize: 1, comment: { hello: 'find' } } + expectResult: *documents + expectEvents: + - client: *client + eventType: command + events: + - commandStartedEvent: + command: + find: *collection + batchSize: 1 + comment: { hello: 'find' } + $db: *db + lsid: { $$exists: true } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + comment: { hello: 'getMore' } + $db: *db + lsid: { $$exists: true } + commandName: getMore + + # Tailable cursor + - description: does not close the cursor when receiving an empty batch + runOnRequirements: + - serverless: forbid + operations: + - name: dropCollection + object: *db + arguments: + collection: &cappedCollection cappedCollection + - name: createCollection + object: *db + arguments: + collection: *cappedCollection + capped: true + size: 4096 + max: 3 + saveResultAsEntity: *cappedCollection + - name: insertMany + object: *cappedCollection + arguments: + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - name: createCommandCursor + object: *db + arguments: + cursorType: tailable + commandName: find + batchSize: 2 + command: { find: *cappedCollection, tailable: true } + saveResultAsEntity: &cursor cursor + - name: iterateOnce + object: *cursor + - name: iterateOnce + object: *cursor + - name: iterateOnce + object: *cursor + - name: close + object: *cursor + expectEvents: + - client: *client + eventType: command + events: + - commandStartedEvent: + command: + drop: *cappedCollection + commandName: drop + - commandStartedEvent: + command: + create: *cappedCollection + commandName: create + - commandStartedEvent: + command: + insert: *cappedCollection + commandName: insert + - commandStartedEvent: + command: + find: *cappedCollection + $db: *db + lsid: { $$exists: true } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *cappedCollection + $db: *db + lsid: { $$exists: true } + commandName: getMore + - commandStartedEvent: + command: + killCursors: *cappedCollection + cursors: { $$type: array } + commandName: killCursors diff --git a/source/unified-test-format/tests/valid-pass/entity-commandCursor.json b/source/unified-test-format/tests/valid-pass/entity-commandCursor.json new file mode 100644 index 0000000000..72b74b4a9a --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/entity-commandCursor.json @@ -0,0 +1,278 @@ +{ + "description": "entity-commandCursor", + "schemaVersion": "1.3", + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "db" + } + }, + { + "collection": { + "id": "collection", + "database": "db", + "collectionName": "collection" + } + } + ], + "initialData": [ + { + "collectionName": "collection", + "databaseName": "db", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "tests": [ + { + "description": "runCursorCommand creates and exhausts cursor by running getMores", + "operations": [ + { + "name": "runCursorCommand", + "object": "db", + "arguments": { + "commandName": "find", + "batchSize": 2, + "command": { + "find": "collection", + "filter": {}, + "batchSize": 2 + } + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "collection", + "filter": {}, + "batchSize": 2, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "collection", + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "createCommandCursor creates a cursor and stores it as an entity that can be iterated one document at a time", + "operations": [ + { + "name": "createCommandCursor", + "object": "db", + "arguments": { + "commandName": "find", + "batchSize": 2, + "command": { + "find": "collection", + "filter": {}, + "batchSize": 2 + } + }, + "saveResultAsEntity": "myRunCommandCursor" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "myRunCommandCursor", + "expectResult": { + "_id": 1, + "x": 11 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "myRunCommandCursor", + "expectResult": { + "_id": 2, + "x": 22 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "myRunCommandCursor", + "expectResult": { + "_id": 3, + "x": 33 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "myRunCommandCursor", + "expectResult": { + "_id": 4, + "x": 44 + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "myRunCommandCursor", + "expectResult": { + "_id": 5, + "x": 55 + } + } + ] + }, + { + "description": "createCommandCursor's cursor can be closed and will perform a killCursors operation", + "operations": [ + { + "name": "createCommandCursor", + "object": "db", + "arguments": { + "commandName": "find", + "batchSize": 2, + "command": { + "find": "collection", + "filter": {}, + "batchSize": 2 + } + }, + "saveResultAsEntity": "myRunCommandCursor" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "myRunCommandCursor", + "expectResult": { + "_id": 1, + "x": 11 + } + }, + { + "name": "close", + "object": "myRunCommandCursor" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "collection", + "filter": {}, + "batchSize": 2, + "$db": "db", + "lsid": { + "$$exists": true + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "killCursors": "collection", + "cursors": { + "$$type": "array" + } + }, + "commandName": "killCursors" + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/entity-commandCursor.yml b/source/unified-test-format/tests/valid-pass/entity-commandCursor.yml new file mode 100644 index 0000000000..3becf2095a --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/entity-commandCursor.yml @@ -0,0 +1,115 @@ +description: entity-commandCursor +schemaVersion: '1.3' +createEntities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent] + - database: + id: &db db + client: *client + databaseName: *db + - collection: + id: &collection collection + database: *db + collectionName: *collection +initialData: + - collectionName: collection + databaseName: *db + documents: &documents + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + - { _id: 5, x: 55 } +tests: + - description: runCursorCommand creates and exhausts cursor by running getMores + operations: + - name: runCursorCommand + object: *db + arguments: + commandName: find + batchSize: 2 + command: { find: *collection, filter: {}, batchSize: 2 } + expectResult: *documents + expectEvents: + - client: *client + events: + - commandStartedEvent: + command: + find: *collection + filter: {} + batchSize: 2 + $db: *db + lsid: { $$exists: true } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + $db: *db + lsid: { $$exists: true } + commandName: getMore + - commandStartedEvent: + command: + getMore: { $$type: [int, long] } + collection: *collection + $db: *db + lsid: { $$exists: true } + commandName: getMore + + - description: createCommandCursor creates a cursor and stores it as an entity that can be iterated one document at a time + operations: + - name: createCommandCursor + object: *db + arguments: + commandName: find + batchSize: 2 + command: { find: *collection, filter: {}, batchSize: 2 } + saveResultAsEntity: &myRunCommandCursor myRunCommandCursor + - name: iterateUntilDocumentOrError + object: *myRunCommandCursor + expectResult: { _id: 1, x: 11 } + - name: iterateUntilDocumentOrError + object: *myRunCommandCursor + expectResult: { _id: 2, x: 22 } + - name: iterateUntilDocumentOrError + object: *myRunCommandCursor + expectResult: { _id: 3, x: 33 } + - name: iterateUntilDocumentOrError + object: *myRunCommandCursor + expectResult: { _id: 4, x: 44 } + - name: iterateUntilDocumentOrError + object: *myRunCommandCursor + expectResult: { _id: 5, x: 55 } + + - description: createCommandCursor's cursor can be closed and will perform a killCursors operation + operations: + - name: createCommandCursor + object: *db + arguments: + commandName: find + batchSize: 2 + command: { find: *collection, filter: {}, batchSize: 2 } + saveResultAsEntity: myRunCommandCursor + - name: iterateUntilDocumentOrError + object: *myRunCommandCursor + expectResult: { _id: 1, x: 11 } + - name: close + object: *myRunCommandCursor + expectEvents: + - client: *client + events: + - commandStartedEvent: + command: + find: *collection + filter: {} + batchSize: 2 + $db: *db + lsid: { $$exists: true } + commandName: find + - commandStartedEvent: + command: + killCursors: *collection + cursors: { $$type: array } + commandName: killCursors diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 56a386493b..f4633e2a55 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -262,6 +262,14 @@ Test runners MUST support the following types of entities: operation. See `Cursor Operations`_ for a list of operations. + +- CommandCursor. These entities are not defined in `createEntities`_ but are + instead created by using `operation.saveResultAsEntity + `_ with a `createCommandCursor`_ + operation. + + See `Cursor Operations`_ for a list of operations. + - Event list. See `storeEventsAsEntities `_. The event list MUST store BSON documents. The type of the list itself is not prescribed @@ -1770,11 +1778,35 @@ runCommand Generic command runner. -This method does not inherit a read concern or write concern (per the -`Read and Write Concern <../read-write-concern/read-write-concern.rst#generic-command-method>`__ -spec), nor does it inherit a read preference (per the +This method does not inherit a read preference (per the +`Server Selection <../server-selection/server-selection.rst#use-of-read-preferences-with-commands>`__ +spec); however, ``readPreference`` may be specified as an argument. + +The following arguments are supported: + +- ``command``: Required document. The command to be executed. + +- ``commandName``: Required string. The name of the command to run. This is used + by languages that are unable preserve the order of keys in the ``command`` + argument when parsing YAML/JSON. + +- ``readPreference``: Optional object. See `commonOptions_readPreference`_. + +- ``session``: Optional string. See `commonOptions_session`_. + +runCursorCommand +~~~~~~~~~~~~~~~~ + +`Generic cursor returning command runner <../run-command/run-command.rst>`__. + +This method does not inherit a read preference (per the `Server Selection <../server-selection/server-selection.rst#use-of-read-preferences-with-commands>`__ -spec); however, they may be specified as arguments. +spec); however, ``readPreference`` may be specified as an argument. + +This operation proxies the database's ``runCursorCommand`` method and supports the same arguments and options (note: handling for `getMore` options may vary by driver implementation). + +When executing the provided command, the test runner MUST fully iterate the cursor. +This will ensure consistent behavior between drivers that eagerly create a server-side cursor and those that do so lazily when iteration begins. The following arguments are supported: @@ -1788,6 +1820,26 @@ The following arguments are supported: - ``session``: Optional string. See `commonOptions_session`_. +- ``batchSize``: Optional positive integer value. Use this value to configure the ``batchSize`` option sent on subsequent ``getMore`` commands. + +- ``maxTimeMS``: Optional non-negative integer value. Use this value to configure the ``maxTimeMS`` option sent on subsequent ``getMore`` commands. + +- ``comment``: Optional BSON value. Use this value to configure the ``comment`` option sent on subsequent ``getMore`` commands. + +- ``cursorType``: Optional string enum value, one of ``'tailable' | 'tailableAwait' | 'nonTailable'``. Use this value to configure the enum passed to the ``cursorType`` option. + +- ``timeoutMode``: Optional string enum value, one of ``'iteration' | 'cursorLifetime'``. Use this value to configure the enum passed to the ``timeoutMode`` option. + +createCommandCursor +~~~~~~~~~~~~~~~~~~~ + +This operation proxies the database's ``runCursorCommand`` method and supports the same arguments and options (note: handling for `getMore` options may vary by driver implementation). +Test runners MUST ensure that the server-side cursor is created (i.e. the command document has executed) as part of this operation and before the resulting cursor might be saved with `operation.saveResultAsEntity `_. +Test runners for drivers that lazily execute the command on the first iteration of the cursor MUST iterate the resulting cursor once. +The result from this iteration MUST be used as the result for the first iteration operation on the cursor. + +Test runners MUST NOT iterate the resulting cursor when executing this operation and test files SHOULD NOT specify `operation.expectResult `_ for this operation. + watch ~~~~~ @@ -2079,7 +2131,7 @@ Cursor Operations There are no defined APIs for change streams and cursors since the mechanisms for iteration may differ between synchronous and asynchronous drivers. To account for this, this section explicitly defines the supported -operations for the ``ChangeStream`` and ``FindCursor`` entity types. +operations for the ``ChangeStream``, ``FindCursor``, and ``CommandCursor`` entity types. Test runners MUST ensure that the iteration operations defined in this section will not inadvertently skip the first document for a cursor. Albeit @@ -3906,6 +3958,8 @@ Changelog .. Please note schema version bumps in changelog entries where applicable. +:2023-05-17: Add ``runCursorCommand`` and ``createCommandCursor`` operations. + Added ``commandCursor`` entity type which can be used with existing cursor operations. :2023-05-12: Deprecate "sharded-replicaset" topology type. Note that server 3.6+ requires replica sets for shards, which is also relevant to load balanced topologies.