Merge branch 'Automatticgh-9850' into refactor/improve-types

History.md

@@ -1,3 +1,14 @@
+5.11.17 / 2021-02-17
+====================
+ * fix(populate): handle `perDocumentLimit` when multiple documents reference the same populated doc #9906
+ * fix(document): handle directly setting embedded document array element with projection #9909
+ * fix(map): cast ObjectId to string inside of MongooseMap #9938 [HunterKohler](https://github.com/HunterKohler)
+ * fix(model): use schema-level default collation for indexes if index doesn't have collation #9912
+ * fix(index.d.ts): make `SchemaTypeOptions#type` optional again to allow alternative typeKeys #9927
+ * fix(index.d.ts): support `{ type: String }` in schema definition when using SchemaDefinitionType generic #9911
+ * docs(populate+schematypes): document the `$*` syntax for populating every entry in a map #9907
+ * docs(connection): clarify that `Connection#transaction()` promise resolves to a command result #9919
+
 5.11.16 / 2021-02-12
 ====================
  * fix(document): skip applying array element setters when init-ing an array #9889

docs/populate.pug

@@ -74,6 +74,7 @@ block content
       <li><a href="#dynamic-ref">Dynamic References via `refPath`</a></li>
       <li><a href="#populate-virtuals">Populate Virtuals</a></li>
       <li><a href="#count">Populate Virtuals: The Count Option</a></li>
+      <li><a href="#populating-maps">Populating Maps</a></li>
       <li><a href="#populate-middleware">Populate in Middleware</a></li>
     </ul>
 
@@ -728,6 +729,76 @@ block content
     doc.numMembers; // 2
     ```
 
+    <h3 id="populating-maps"><a href="#populating-maps">Populating Maps</a></h3>
+
+    [Maps](/docs/schematypes.html#maps) are a type that represents an object with arbitrary
+    string keys. For example, in the below schema, `members` is a map from strings to ObjectIds.
+
+    ```javascript
+    const BandSchema = new Schema({
+      name: String,
+      members: {
+        type: Map,
+        of: {
+          type: 'ObjectId',
+          ref: 'Person'
+        }
+      }
+    });
+    const Band = mongoose.model('Band', bandSchema);
+    ```
+
+    This map has a `ref`, which means you can use `populate()` to populate all the ObjectIds
+    in the map. Suppose you have the below `band` document:
+
+    ```
+    const person1 = new Person({ name: 'Vince Neil' });
+    const person2 = new Person({ name: 'Mick Mars' });
+
+    const band = new Band({
+      name: 'Motley Crue',
+      members: {
+        'singer': person1._id,
+        'guitarist': person2._id
+      }
+    });
+    ```
+
+    You can `populate()` every element in the map by populating the special path `members.$*`.
+    `$*` is a special syntax that tells Mongoose to look at every key in the map.
+
+    ```javascript
+    const band = await Band.findOne({ name: 'Motley Crue' }).populate('members.$*');
+
+    band.members.get('singer'); // { _id: ..., name: 'Vince Neil' }
+    ```
+
+    You can also populate paths in maps of subdocuments using `$*`. For example, suppose you
+    have the below `librarySchema`:
+
+    ```javascript
+    const librarySchema = new Schema({
+      name: String,
+      books: {
+        type: Map,
+        of: new Schema({
+          title: String,
+          author: {
+            type: 'ObjectId',
+            ref: 'Person'
+          }
+        })
+      }
+    });
+    const Library = mongoose.model('Library, librarySchema');
+    ```
+
+    You can `populate()` every book's author by populating `books.$*.author`:
+
+    ```javascript
+    const libraries = await Library.find().populate('books.$*.author');
+    ```
+
     <h3 id="populate-middleware"><a href="#populate-middleware">Populate in Middleware</a></h3>
 
     You can populate in either pre or post [hooks](http://mongoosejs.com/docs/middleware.html). If you want to

docs/schematypes.pug

@@ -608,6 +608,32 @@ block content
     Keys in a BSON object are ordered, so this means the [insertion order](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map#Description)
     property of maps is maintained.
 
+    Mongoose supports a special `$*` syntax to [populate](/docs/populate.html) all elements in a map.
+    For example, suppose your `socialMediaHandles` map contains a `ref`:
+
+    ```javascript
+    const userSchema = new Schema({
+      socialMediaHandles: {
+        type: Map,
+        of: new Schema({
+          handle: String,
+          oauth: {
+            type: ObjectId,
+            ref: 'OAuth'
+          }
+        })
+      }
+    });
+    const User = mongoose.model('User', userSchema);
+    ```
+
+    To populate every `socialMediaHandles` entry's `oauth` property, you should populate
+    on `socialMediaHandles.$*.oauth`:
+
+    ```javascript
+    const user = await User.findOne().populate('socialMediaHandles.$*.oauth');
+    ```
+
     <h3 id="getters"><a href="#getters">Getters</a></h3>
 
     Getters are like virtuals for paths defined in your schema. For example,

docs/source/api.js

@@ -164,7 +164,7 @@ function parse() {
 
       ctx.description = prop.description.full.
         replace(/<br \/>/ig, ' ').
-        replace(/&gt;/i, '>');
+        replace(/&gt;/ig, '>');
       ctx.description = highlight(ctx.description);
 
       data.props.push(ctx);

index.d.ts

@@ -567,9 +567,11 @@ declare module 'mongoose' {
 
     /** The return value of this method is used in calls to JSON.stringify(doc). */
     toJSON(options?: ToObjectOptions): LeanDocument<this>;
+    toJSON<T>(options?: ToObjectOptions): T;
 
     /** Converts this document into a plain-old JavaScript object ([POJO](https://masteringjs.io/tutorials/fundamentals/pojo)). */
     toObject(options?: ToObjectOptions): LeanDocument<this>;
+    toObject<T>(options?: ToObjectOptions): T;
 
     /** Clears the modified state on the specified path. */
     unmarkModified(path: string): void;
@@ -651,11 +653,15 @@ declare module 'mongoose' {
     countDocuments(filter: FilterQuery<T>, callback?: (err: any, count: number) => void): Query<number, T, TQueryHelpers>;
 
     /** Creates a new document or documents */
-    create<DocContents = T | DocumentDefinition<T>>(docs: CreateDoc<DocContents>[], options?: SaveOptions): Promise<T[]>;
-    create<DocContents = T | DocumentDefinition<T>>(doc: CreateDoc<DocContents>): Promise<T>;
-    create<DocContents = T | DocumentDefinition<T>>(...docs: CreateDoc<DocContents>[]): Promise<T[]>;
-    create<DocContents = T | DocumentDefinition<T>>(docs: CreateDoc<DocContents>[], callback: (err: CallbackError, docs: T[]) => void): void;
-    create<DocContents = T | DocumentDefinition<T>>(doc: CreateDoc<DocContents>, callback: (err: CallbackError, doc: T) => void): void;
+    create(doc: T | DocumentDefinition<T>): Promise<T>;
+    create(docs: (T | DocumentDefinition<T>)[], options?: SaveOptions): Promise<T[]>;
+    create(docs: (T | DocumentDefinition<T>)[], callback: (err: CallbackError, docs: T[]) => void): void;
+    create(doc: T | DocumentDefinition<T>, callback: (err: CallbackError, doc: T) => void): void;
+    create<DocContents = T | DocumentDefinition<T>>(docs: DocContents[], options?: SaveOptions): Promise<T[]>;
+    create<DocContents = T | DocumentDefinition<T>>(doc: DocContents): Promise<T>;
+    create<DocContents = T | DocumentDefinition<T>>(...docs: DocContents[]): Promise<T[]>;
+    create<DocContents = T | DocumentDefinition<T>>(docs: DocContents[], callback: (err: CallbackError, docs: T[]) => void): void;
+    create<DocContents = T | DocumentDefinition<T>>(doc: DocContents, callback: (err: CallbackError, doc: T) => void): void;
 
     /**
      * Create the collection for this model. By default, if no indexes are specified,
@@ -680,14 +686,14 @@ declare module 'mongoose' {
      * Behaves like `remove()`, but deletes all documents that match `conditions`
      * regardless of the `single` option.
      */
-    deleteMany(filter?: any, options?: QueryOptions, callback?: (err: CallbackError) => void): Query<any, T, TQueryHelpers>;
+    deleteMany(filter?: FilterQuery<T>, options?: QueryOptions, callback?: (err: CallbackError) => void): Query<mongodb.DeleteWriteOpResultObject['result'] & { deletedCount?: number }, T, TQueryHelpers>;
 
     /**
      * Deletes the first document that matches `conditions` from the collection.
      * Behaves like `remove()`, but deletes at most one document regardless of the
      * `single` option.
      */
-    deleteOne(filter?: any, options?: QueryOptions, callback?: (err: CallbackError) => void): Query<any, T, TQueryHelpers>;
+    deleteOne(filter?: FilterQuery<T>, options?: QueryOptions, callback?: (err: CallbackError) => void): Query<mongodb.DeleteWriteOpResultObject['result'] & { deletedCount?: number }, T, TQueryHelpers>;
 
     /**
      * Sends `createIndex` commands to mongo for each index declared in the schema.
@@ -856,13 +862,13 @@ declare module 'mongoose' {
      * @deprecated use `updateOne` or `updateMany` instead.
      * Creates a `update` query: updates one or many documents that match `filter` with `update`, based on the `multi` option.
      */
-    update(filter?: FilterQuery<T>, update?: UpdateQuery<T>, options?: QueryOptions | null, callback?: (err: any, res: any) => void): Query<any, T, TQueryHelpers>;
+    update(filter?: FilterQuery<T>, update?: UpdateQuery<T>, options?: QueryOptions | null, callback?: (err: any, res: any) => void): Query<mongodb.WriteOpResult['result'], T, TQueryHelpers>;
 
     /** Creates a `updateMany` query: updates all documents that match `filter` with `update`. */
-    updateMany(filter?: FilterQuery<T>, update?: UpdateQuery<T>, options?: QueryOptions | null, callback?: (err: any, res: any) => void): Query<any, T, TQueryHelpers>;
+    updateMany(filter?: FilterQuery<T>, update?: UpdateQuery<T>, options?: QueryOptions | null, callback?: (err: any, res: any) => void): Query<mongodb.UpdateWriteOpResult['result'], T, TQueryHelpers>;
 
     /** Creates a `updateOne` query: updates the first document that matches `filter` with `update`. */
-    updateOne(filter?: FilterQuery<T>, update?: UpdateQuery<T>, options?: QueryOptions | null, callback?: (err: any, res: any) => void): Query<any, T, TQueryHelpers>;
+    updateOne(filter?: FilterQuery<T>, update?: UpdateQuery<T>, options?: QueryOptions | null, callback?: (err: any, res: any) => void): Query<mongodb.UpdateWriteOpResult['result'], T, TQueryHelpers>;
 
     /** Creates a Query, applies the passed conditions, and returns the Query. */
     where(path: string, val?: any): Query<Array<T>, T, TQueryHelpers>;
@@ -1203,14 +1209,15 @@ declare module 'mongoose' {
     virtualpath(name: string): VirtualType | null;
   }
 
-  type SchemaDefinitionWithBuiltInClass<T> = T extends number
+  type SchemaDefinitionWithBuiltInClass<T extends number | string | Function> = T extends number
     ? (typeof Number | 'number' | 'Number')
     : T extends string
     ? (typeof String | 'string' | 'String')
     : (Function | string);
 
-  type SchemaDefinitionProperty<T = undefined> = SchemaTypeOptions<T extends undefined ? any : T> |
-    SchemaDefinitionWithBuiltInClass<T> |
+  type SchemaDefinitionProperty<T = undefined> = T extends string | number | Function
+    ? (SchemaDefinitionWithBuiltInClass<T> | SchemaTypeOptions<T>) :
+    SchemaTypeOptions<T extends undefined ? any : T> |
     typeof SchemaType |
     Schema<T extends Document ? T : Document<any>> |
     Schema<T extends Document ? T : Document<any>>[] |
@@ -1386,7 +1393,7 @@ declare module 'mongoose' {
   }
 
   interface SchemaTypeOptions<T> {
-    type: T;
+    type?: T extends string | number | Function ? SchemaDefinitionWithBuiltInClass<T> : T;
 
     /** Defines a virtual with the given name that gets/sets this path. */
     alias?: string;
@@ -1515,6 +1522,26 @@ declare module 'mongoose' {
     [other: string]: any;
   }
 
+  export type RefType =
+    | number
+    | string
+    | Buffer
+    | undefined
+    | mongoose.Types.ObjectId
+    | mongoose.Types.Buffer
+    | typeof mongoose.Schema.Types.Number
+    | typeof mongoose.Schema.Types.String
+    | typeof mongoose.Schema.Types.Buffer
+    | typeof mongoose.Schema.Types.ObjectId;
+
+  /**
+   * Reference another Model
+   */
+  export type PopulatedDoc<
+    PopulatedType,
+    RawId extends RefType = (PopulatedType extends { _id?: RefType; } ? NonNullable<PopulatedType['_id']> : mongoose.Types.ObjectId) | undefined
+    > = PopulatedType | RawId;
+
   interface IndexOptions {
     background?: boolean,
     expires?: number | string
@@ -1742,6 +1769,7 @@ declare module 'mongoose' {
 
       /** Returns a native js Array. */
       toObject(options?: ToObjectOptions): any;
+      toObject<T>(options?: ToObjectOptions): T;
 
       /** Wraps [`Array#unshift`](https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/Array/unshift) with proper change tracking. */
       unshift(...args: any[]): number;
@@ -2669,4 +2697,4 @@ declare module 'mongoose' {
 
   /* for ts-mongoose */
   class mquery {}
-}
+}

lib/connection.js

@@ -469,7 +469,7 @@ Connection.prototype.startSession = _wrapConnHelper(function startSession(option
  * @method transaction
  * @param {Function} fn Function to execute in a transaction
  * @param {mongodb.TransactionOptions} [options] Optional settings for the transaction
- * @return {Promise<Any>} promise that resolves to the returned value of `fn`
+ * @return {Promise<Any>} promise that is fulfilled if Mongoose successfully committed the transaction, or rejects if the transaction was aborted or if Mongoose failed to commit the transaction. If fulfilled, the promise resolves to a MongoDB command result.
  * @api public
  */
 
@@ -832,7 +832,6 @@ Connection.prototype.openUri = function(uri, options, callback) {
     _this.client = client;
     client.connect((error) => {
       if (error) {
-        _this.readyState = STATES.disconnected;
         return reject(error);
       }
 
@@ -846,6 +845,7 @@ Connection.prototype.openUri = function(uri, options, callback) {
   this.$initialConnection = Promise.all([promise, parsePromise]).
     then(res => res[0]).
     catch(err => {
+      this.readyState = STATES.disconnected;
       if (err != null && err.name === 'MongoServerSelectionError') {
         err = serverSelectionError.assimilateError(err);
       }
@@ -858,7 +858,7 @@ Connection.prototype.openUri = function(uri, options, callback) {
   this.then = function(resolve, reject) {
     return this.$initialConnection.then(() => {
       if (typeof resolve === 'function') {
-        resolve(_this);
+        return resolve(_this);
       }
     }, reject);
   };
@@ -940,14 +940,15 @@ function _setClient(conn, client, options, dbName) {
   }
 
   // Backwards compat for mongoose 4.x
-  db.on('reconnect', function() {
-    _handleReconnect();
-  });
   db.s.topology.on('reconnectFailed', function() {
     conn.emit('reconnectFailed');
   });
 
   if (!options.useUnifiedTopology) {
+    db.on('reconnect', function() {
+      _handleReconnect();
+    });
+
     db.s.topology.on('left', function(data) {
       conn.emit('left', data);
     });
@@ -963,11 +964,16 @@ function _setClient(conn, client, options, dbName) {
       conn.emit('attemptReconnect');
     });
   }
-  if (!options.useUnifiedTopology || !type.startsWith('ReplicaSet')) {
+  if (!options.useUnifiedTopology) {
     db.on('close', function() {
       // Implicitly emits 'disconnected'
       conn.readyState = STATES.disconnected;
     });
+  } else if (!type.startsWith('ReplicaSet')) {
+    client.on('close', function() {
+      // Implicitly emits 'disconnected'
+      conn.readyState = STATES.disconnected;
+    });
   }
 
   if (!options.useUnifiedTopology) {
@@ -977,11 +983,11 @@ function _setClient(conn, client, options, dbName) {
         conn.readyState = STATES.disconnected;
       }
     });
-  }
 
-  db.on('timeout', function() {
-    conn.emit('timeout');
-  });
+    db.on('timeout', function() {
+      conn.emit('timeout');
+    });
+  }
 
   delete conn.then;
   delete conn.catch;

lib/helpers/populate/assignVals.js

@@ -240,7 +240,7 @@ function valueFilter(val, assignmentOpts, populateOptions) {
   if (populateOptions.justOne === false) {
     return [];
   }
-  return val;
+  return val == null ? val : null;
 }
 
 /*!

lib/model.js

@@ -4592,7 +4592,12 @@ function _assign(model, vals, mod, assignmentOpts) {
           if (Array.isArray(rawDocs[key])) {
             rawDocs[key].push(val);
             rawOrder[key].push(i);
-          } else {
+          } else if (isVirtual ||
+            rawDocs[key].constructor !== val.constructor ||
+            String(rawDocs[key]._id) !== String(val._id)) {
+            // May need to store multiple docs with the same id if there's multiple models
+            // if we have discriminators or a ref function. But avoid converting to an array
+            // if we have multiple queries on the same model because of `perDocumentLimit` re: gh-9906
             rawDocs[key] = [rawDocs[key], val];
             rawOrder[key] = [rawOrder[key], i];
           }