Merge branch 'master' into #11532

.eslintrc.json

@@ -7,28 +7,40 @@
         "dist",
         "test/files/*"
     ],
-    "overrides": [
-        {
-            "files": [
-                "**/*.{ts,tsx}"
-            ],
-            "extends": [
-                "plugin:@typescript-eslint/eslint-recommended",
-                "plugin:@typescript-eslint/recommended"
-            ],
-            "plugins": [
-                "@typescript-eslint"
-            ],
-            "rules": {
-                "@typescript-eslint/triple-slash-reference": "off",
-                "spaced-comment": ["error", "always", { "markers": ["/"] }],
-                "@typescript-eslint/no-explicit-any": "off",
-                "@typescript-eslint/ban-types": "off",
-                "@typescript-eslint/no-unused-vars": "off",
-                "@typescript-eslint/explicit-module-boundary-types": "off"
-            }
+    "overrides": [{
+        "files": [
+            "**/*.{ts,tsx}"
+        ],
+        "extends": [
+            "plugin:@typescript-eslint/eslint-recommended",
+            "plugin:@typescript-eslint/recommended"
+        ],
+        "plugins": [
+            "@typescript-eslint"
+        ],
+        "rules": {
+            "@typescript-eslint/triple-slash-reference": "off",
+            "spaced-comment": ["error", "always", {
+                "markers": ["/"]
+            }],
+            "@typescript-eslint/no-explicit-any": "off",
+            "@typescript-eslint/ban-types": "off",
+            "@typescript-eslint/no-unused-vars": "off",
+            "@typescript-eslint/explicit-module-boundary-types": "off",
+            "@typescript-eslint/indent": ["error", 2, {
+                "SwitchCase": 1
+            }],
+            "@typescript-eslint/prefer-optional-chain": "error",
+            "@typescript-eslint/brace-style": "error",
+            "@typescript-eslint/no-dupe-class-members": "error",
+            "@typescript-eslint/no-redeclare": "error",
+            "@typescript-eslint/type-annotation-spacing": "error",
+            "@typescript-eslint/object-curly-spacing": ["error", "always"],
+            "@typescript-eslint/semi": "error",
+            "@typescript-eslint/space-before-function-paren": ["error", "never"],
+            "@typescript-eslint/space-infix-ops": "error"
         }
-    ],
+    }],
     "plugins": [
         "mocha-no-only"
     ],
@@ -148,4 +160,4 @@
         ],
         "no-empty": "off"
     }
-}
+}

docs/change-streams.md

@@ -0,0 +1,86 @@
+## Change Streams
+
+[Change streams](https://www.mongodb.com/developer/quickstart/nodejs-change-streams-triggers/) let you listen for updates to documents in a given model's collection, or even documents in an entire database.
+Unlike [middleware](/docs/middleware.html), change streams are a MongoDB server construct, which means they pick up changes from anywhere.
+Even if you update a document from a MongoDB GUI, your Mongoose change stream will be notified.
+
+The `watch()` function creates a change stream.
+Change streams emit a `'data'` event when a document is updated.
+
+```javascript
+const Person = mongoose.model('Person', new mongoose.Schema({ name: String }));
+
+// Create a change stream. The 'change' event gets emitted when there's a
+// change in the database. Print what the change stream emits.
+Person.watch().
+  on('change', data => console.log(data));
+
+// Insert a doc, will trigger the change stream handler above
+await Person.create({ name: 'Axl Rose' });
+```
+
+The above script will print output that looks like:
+
+```
+{
+  _id: {
+    _data: '8262408DAC000000012B022C0100296E5A10042890851837DB4792BE6B235E8B85489F46645F6964006462408DAC6F5C42FF5EE087A20004'
+  },
+  operationType: 'insert',
+  clusterTime: new Timestamp({ t: 1648397740, i: 1 }),
+  fullDocument: {
+    _id: new ObjectId("62408dac6f5c42ff5ee087a2"),
+    name: 'Axl Rose',
+    __v: 0
+  },
+  ns: { db: 'test', coll: 'people' },
+  documentKey: { _id: new ObjectId("62408dac6f5c42ff5ee087a2") }
+}
+```
+
+Note that you **must** be connected to a MongoDB replica set or sharded cluster to use change streams.
+If you try to call `watch()` when connected to a standalone MongoDB server, you'll get the below error.
+
+```
+MongoServerError: The $changeStream stage is only supported on replica sets
+```
+
+If you're using `watch()` in production, we recommend using [MongoDB Atlas](https://www.mongodb.com/atlas/database).
+For local development, we recommend [mongodb-memory-server](https://www.npmjs.com/package/mongodb-memory-server) or [run-rs](https://www.npmjs.com/package/run-rs) to start a replica set locally.
+
+### Iterating using `next()`
+
+If you want to iterate through a change stream in a [AWS Lambda function](/docs/lambda.html), do **not** use event emitters to listen to the change stream.
+You need to make sure you close your change stream when your Lambda function is done executing, because your change stream may end up in an inconsistent state if Lambda stops your container while the change stream is pulling data from MongoDB.
+
+Change streams also have a `next()` function that lets you explicitly wait for the next change to come in.
+Use `resumeAfter` to track where the last change stream left off, and add a timeout to make sure your handler doesn't wait forever if no changes come in.
+
+```javascript
+let resumeAfter = undefined;
+
+exports.handler = async (event, context) => {
+  // add this so that we can re-use any static/global variables between function calls if Lambda
+  // happens to re-use existing containers for the invocation.
+  context.callbackWaitsForEmptyEventLoop = false;
+
+  await connectToDatabase();
+
+  // Use MongoDB Node driver's `watch()` function, because Mongoose change streams
+  // don't support `next()` yet. See https://github.com/Automattic/mongoose/issues/11527
+  const changeStream = await Country.watch([], { resumeAfter });
+
+  // Change stream `next()` will wait forever if there are no changes. So make sure to
+  // stop listening to the change stream after a fixed period of time.
+  let timeoutPromise = new Promise(resolve => setTimeout(() => resolve(false), 1000));
+  let doc = null;
+  while (doc = await Promise.race([changeStream.next(), timeoutPromise])) {
+    console.log('Got', doc);
+  }
+
+  // `resumeToken` tells you where the change stream left off, so next function instance
+  // can pick up any changes that happened in the meantime.
+  resumeAfter = changeStream.resumeToken;
+  await changeStream.close();
+};
+```

docs/lambda.md

@@ -30,7 +30,7 @@ exports.handler = async function(event, context) {
 
     // `await`ing connection after assigning to the `conn` variable
     // to avoid multiple function calls creating new connections
-    await conn;
+    await conn.asPromise();
     conn.model('Test', new mongoose.Schema({ name: String }));
   }
 
@@ -65,7 +65,7 @@ exports.connect = async function() {
 
     // `await`ing connection after assigning to the `conn` variable
     // to avoid multiple function calls creating new connections
-    await conn;
+    await conn.asPromise();
   }
 
   return conn;

docs/layout.pug

@@ -74,6 +74,9 @@ html(lang='en')
                   a.pure-menu-link(href="/docs/tutorials/ssl.html", class=outputUrl === '/docs/tutorials/ssl.html' ? 'selected' : '') SSL Connections
               li.pure-menu-item.sub-item
                 a.pure-menu-link(href="/docs/models.html", class=outputUrl === '/docs/models.html' ? 'selected' : '') Models
+              - if (['/docs/models', '/docs/change-streams'].some(path => outputUrl.startsWith(path)))
+                li.pure-menu-item.tertiary-item
+                  a.pure-menu-link(href="/docs/change-streams.html", class=outputUrl === '/docs/change-streams.html' ? 'selected' : '') Change Streams
               li.pure-menu-item.sub-item
                 a.pure-menu-link(href="/docs/documents.html", class=outputUrl === '/docs/documents.html' ? 'selected' : '') Documents
               li.pure-menu-item.sub-item

docs/source/index.js

@@ -62,6 +62,7 @@ exports['docs/jobs.pug'] = {
   title: 'Mongoose MongoDB Jobs',
   jobs
 };
+exports['docs/change-streams.md'] = { title: 'MongoDB Change Streams in NodeJS with Mongoose', markdown: true };
 
 for (const props of Object.values(exports)) {
   props.jobs = jobs;

docs/tutorials/ssl.md

@@ -68,3 +68,27 @@ MongooseServerSelectionError: Hostname/IP does not match certificate's altnames:
 
 The SSL certificate's [common name](https://knowledge.digicert.com/solution/SO7239.html) **must** line up with the host name
 in your connection string. If the SSL certificate is for `hostname2.mydomain.com`, your connection string must connect to `hostname2.mydomain.com`, not any other hostname or IP address that may be equivalent to `hostname2.mydomain.com`. For replica sets, this also means that the SSL certificate's common name must line up with the [machine's `hostname`](/docs/connections.html#replicaset-hostnames).
+
+## X509 Auth
+
+If you're using [X509 authentication](https://www.mongodb.com/docs/drivers/node/current/fundamentals/authentication/mechanisms/#x.509), you should set the user name in the connection string, **not** the `connect()` options.
+
+```javascript
+// Do this:
+const username = 'myusername';
+await mongoose.connect(`mongodb://${encodeURIComponent(username)}@localhost:27017/test`, {
+  ssl: true,
+  sslValidate: true,
+  sslCA: `${__dirname}/rootCA.pem`,
+  authMechanism: 'MONGODB-X509'
+});
+
+// Not this:
+await mongoose.connect(`mongodb://localhost:27017/test`, {
+  ssl: true,
+  sslValidate: true,
+  sslCA: `${__dirname}/rootCA.pem`,
+  authMechanism: 'MONGODB-X509'.
+  auth: { username }
+});
+```

lib/document.js

@@ -2506,12 +2506,32 @@ function _getPathsToValidate(doc) {
     }
   }
 
+  for (const path of paths) {
+    const _pathType = doc.$__schema.path(path);
+    if (!_pathType) {
+      continue;
+    }
+
+    // Optimization: if primitive path with no validators, or array of primitives
+    // with no validators, skip validating this path entirely.
+    if (!_pathType.caster && _pathType.validators.length === 0) {
+      paths.delete(path);
+    } else if (_pathType.$isMongooseArray &&
+      !_pathType.$isMongooseDocumentArray && // Skip document arrays...
+      !_pathType.$embeddedSchemaType.$isMongooseArray && // and arrays of arrays
+      _pathType.validators.length === 0 && // and arrays with top-level validators
+      _pathType.$embeddedSchemaType.validators.length === 0) {
+      paths.delete(path);
+    }
+  }
+
   // from here on we're not removing items from paths
 
   // gh-661: if a whole array is modified, make sure to run validation on all
   // the children as well
   for (const path of paths) {
     const _pathType = doc.$__schema.path(path);
+
     if (!_pathType ||
         !_pathType.$isMongooseArray ||
         // To avoid potential performance issues, skip doc arrays whose children

lib/query.js

@@ -39,6 +39,31 @@ const utils = require('./utils');
 const validOps = require('./helpers/query/validOps');
 const wrapThunk = require('./helpers/query/wrapThunk');
 
+const queryOptionMethods = new Set([
+  'allowDiskUse',
+  'batchSize',
+  'collation',
+  'comment',
+  'explain',
+  'hint',
+  'j',
+  'lean',
+  'limit',
+  'maxScan',
+  'maxTimeMS',
+  'maxscan',
+  'projection',
+  'read',
+  'select',
+  'skip',
+  'slice',
+  'sort',
+  'tailable',
+  'w',
+  'writeConcern',
+  'wtimeout'
+]);
+
 /**
  * Query constructor used for building queries. You do not need
  * to instantiate a `Query` directly. Instead use Model functions like
@@ -1630,7 +1655,19 @@ Query.prototype.setOptions = function(options, overwrite) {
     }
   }
 
-  return Query.base.setOptions.call(this, options);
+  // set arbitrary options
+  for (const key of Object.keys(options)) {
+    if (queryOptionMethods.has(key)) {
+      const args = Array.isArray(options[key]) ?
+        options[key] :
+        [options[key]];
+      this[key].apply(this, args);
+    } else {
+      this.options[key] = options[key];
+    }
+  }
+
+  return this;
 };
 
 /**
@@ -5696,4 +5733,4 @@ Query.prototype.model;
  * Export
  */
 
-module.exports = Query;
+module.exports = Query;

test/query.test.js

@@ -3933,4 +3933,16 @@ describe('Query', function() {
     res = await Test.find({ names: { $not: { $regex: 'foo' } } });
     assert.deepStrictEqual(res.map(el => el.names), [['bar']]);
   });
+  it('adding `exec` option does not affect the query (gh-11416)', async() => {
+    const userSchema = new Schema({
+      name: { type: String }
+    });
+
+
+    const User = db.model('User', userSchema);
+    const createdUser = await User.create({ name: 'Hafez' });
+    const users = await User.find({ _id: createdUser._id }).setOptions({ exec: false });
+
+    assert.ok(users.length, 1);
+  });
 });

test/types/aggregate.test.ts

@@ -33,10 +33,18 @@ async function run() {
   }
 
   function eachAsync(): void {
-    Test.aggregate().cursor().eachAsync((doc) => {expectType<any>(doc);});
-    Test.aggregate().cursor().eachAsync((docs) => {expectType<any[]>(docs);}, { batchSize: 2 });
-    Test.aggregate().cursor<ITest>().eachAsync((doc) => {expectType<ITest>(doc);});
-    Test.aggregate().cursor<ITest>().eachAsync((docs) => {expectType<ITest[]>(docs);}, { batchSize: 2 });
+    Test.aggregate().cursor().eachAsync((doc) => {
+      expectType<any>(doc);
+    });
+    Test.aggregate().cursor().eachAsync((docs) => {
+      expectType<any[]>(docs);
+    }, { batchSize: 2 });
+    Test.aggregate().cursor<ITest>().eachAsync((doc) => {
+      expectType<ITest>(doc);
+    });
+    Test.aggregate().cursor<ITest>().eachAsync((docs) => {
+      expectType<ITest[]>(docs);
+    }, { batchSize: 2 });
   }
 
   // Aggregate.prototype.sort()

test/types/connect.test.ts

@@ -7,6 +7,12 @@ expectType<Promise<typeof mongoose>>(connect('mongodb://localhost:27017/test', {
 expectType<Promise<typeof mongoose>>(connect('mongodb://localhost:27017/test', { bufferCommands: true }));
 
 // Callback
-expectType<void>(connect('mongodb://localhost:27017/test', (err: Error | null) => { return; }));
-expectType<void>(connect('mongodb://localhost:27017/test', {}, (err: Error | null) => { return; }));
-expectType<void>(connect('mongodb://localhost:27017/test', { bufferCommands: true }, (err: Error | null) => { return; }));
+expectType<void>(connect('mongodb://localhost:27017/test', (err: Error | null) => {
+  return;
+}));
+expectType<void>(connect('mongodb://localhost:27017/test', {}, (err: Error | null) => {
+  return;
+}));
+expectType<void>(connect('mongodb://localhost:27017/test', { bufferCommands: true }, (err: Error | null) => {
+  return;
+}));