Merge pull request #14724 from Automattic/vkarpov15/gh-14719

Performance improvements for `insertMany()`
Automattic · Jul 10, 2024 · d30a3b9 · d30a3b9
2 parents 7d742e2 + 8223f91
commit d30a3b9
Show file tree

Hide file tree

Showing 5 changed files with 130 additions and 40 deletions.
diff --git a/benchmarks/insertManySimple.js b/benchmarks/insertManySimple.js
@@ -0,0 +1,38 @@
+'use strict';
+
+const mongoose = require('../');
+
+run().catch(err => {
+  console.error(err);
+  process.exit(-1);
+});
+
+async function run() {
+  await mongoose.connect('mongodb://127.0.0.1:27017/mongoose_benchmark');
+  const FooSchema = new mongoose.Schema({ foo: String });
+  const FooModel = mongoose.model('Foo', FooSchema);
+
+  if (!process.env.MONGOOSE_BENCHMARK_SKIP_SETUP) {
+    await FooModel.deleteMany({});
+  }
+
+  const numDocs = 1500;
+  const docs = [];
+  for (let i = 0; i < numDocs; ++i) {
+    docs.push({ foo: 'test foo ' + i });
+  }
+
+  const numIterations = 200;
+  const insertStart = Date.now();
+  for (let i = 0; i < numIterations; ++i) {
+    await FooModel.insertMany(docs);
+  }
+  const insertEnd = Date.now();
+
+  const results = {
+    'Average insertMany time ms': +((insertEnd - insertStart) / numIterations).toFixed(2)
+  };
+
+  console.log(JSON.stringify(results, null, '  '));
+  process.exit(0);
+}
diff --git a/lib/document.js b/lib/document.js
@@ -31,6 +31,7 @@ const getKeysInSchemaOrder = require('./helpers/schema/getKeysInSchemaOrder');
 const getSubdocumentStrictValue = require('./helpers/schema/getSubdocumentStrictValue');
 const handleSpreadDoc = require('./helpers/document/handleSpreadDoc');
 const immediate = require('./helpers/immediate');
+const isBsonType = require('./helpers/isBsonType');
 const isDefiningProjection = require('./helpers/projection/isDefiningProjection');
 const isExclusive = require('./helpers/projection/isExclusive');
 const isPathExcluded = require('./helpers/projection/isPathExcluded');
@@ -2611,17 +2612,6 @@ Document.prototype.validate = async function validate(pathsToValidate, options)
   let parallelValidate;
   this.$op = 'validate';
 
-  if (this.$isSubdocument != null) {
-    // Skip parallel validate check for subdocuments
-  } else if (this.$__.validating) {
-    parallelValidate = new ParallelValidateError(this, {
-      parentStack: options && options.parentStack,
-      conflictStack: this.$__.validating.stack
-    });
-  } else {
-    this.$__.validating = new ParallelValidateError(this, { parentStack: options && options.parentStack });
-  }
-
   if (arguments.length === 1) {
     if (typeof arguments[0] === 'object' && !Array.isArray(arguments[0])) {
       options = arguments[0];
@@ -2632,6 +2622,18 @@ Document.prototype.validate = async function validate(pathsToValidate, options)
     const isOnePathOnly = options.pathsToSkip.indexOf(' ') === -1;
     options.pathsToSkip = isOnePathOnly ? [options.pathsToSkip] : options.pathsToSkip.split(' ');
   }
+  const _skipParallelValidateCheck = options && options._skipParallelValidateCheck;
+
+  if (this.$isSubdocument != null) {
+    // Skip parallel validate check for subdocuments
+  } else if (this.$__.validating && !_skipParallelValidateCheck) {
+    parallelValidate = new ParallelValidateError(this, {
+      parentStack: options && options.parentStack,
+      conflictStack: this.$__.validating.stack
+    });
+  } else if (!_skipParallelValidateCheck) {
+    this.$__.validating = new ParallelValidateError(this, { parentStack: options && options.parentStack });
+  }
 
   if (parallelValidate != null) {
     throw parallelValidate;
@@ -3480,31 +3482,33 @@ Document.prototype.$__reset = function reset() {
   let _this = this;
 
   // Skip for subdocuments
-  const subdocs = this.$parent() === this ? this.$getAllSubdocs() : [];
-  const resetArrays = new Set();
-  for (const subdoc of subdocs) {
-    const fullPathWithIndexes = subdoc.$__fullPathWithIndexes();
-    subdoc.$__reset();
-    if (this.isModified(fullPathWithIndexes) || isParentInit(fullPathWithIndexes)) {
-      if (subdoc.$isDocumentArrayElement) {
-        resetArrays.add(subdoc.parentArray());
-      } else {
-        const parent = subdoc.$parent();
-        if (parent === this) {
-          this.$__.activePaths.clearPath(subdoc.$basePath);
-        } else if (parent != null && parent.$isSubdocument) {
-          // If map path underneath subdocument, may end up with a case where
-          // map path is modified but parent still needs to be reset. See gh-10295
-          parent.$__reset();
+  const subdocs = !this.$isSubdocument ? this.$getAllSubdocs() : null;
+  if (subdocs && subdocs.length > 0) {
+    const resetArrays = new Set();
+    for (const subdoc of subdocs) {
+      const fullPathWithIndexes = subdoc.$__fullPathWithIndexes();
+      subdoc.$__reset();
+      if (this.isModified(fullPathWithIndexes) || isParentInit(fullPathWithIndexes)) {
+        if (subdoc.$isDocumentArrayElement) {
+          resetArrays.add(subdoc.parentArray());
+        } else {
+          const parent = subdoc.$parent();
+          if (parent === this) {
+            this.$__.activePaths.clearPath(subdoc.$basePath);
+          } else if (parent != null && parent.$isSubdocument) {
+            // If map path underneath subdocument, may end up with a case where
+            // map path is modified but parent still needs to be reset. See gh-10295
+            parent.$__reset();
+          }
         }
       }
     }
-  }
 
-  for (const array of resetArrays) {
-    this.$__.activePaths.clearPath(array.$path());
-    array[arrayAtomicsBackupSymbol] = array[arrayAtomicsSymbol];
-    array[arrayAtomicsSymbol] = {};
+    for (const array of resetArrays) {
+      this.$__.activePaths.clearPath(array.$path());
+      array[arrayAtomicsBackupSymbol] = array[arrayAtomicsSymbol];
+      array[arrayAtomicsSymbol] = {};
+    }
   }
 
   function isParentInit(path) {
@@ -3809,6 +3813,8 @@ Document.prototype.$__handleReject = function handleReject(err) {
 Document.prototype.$toObject = function(options, json) {
   const defaultOptions = this.$__schema._defaultToObjectOptions(json);
 
+  const hasOnlyPrimitiveValues = this.$__hasOnlyPrimitiveValues();
+
   // If options do not exist or is not an object, set it to empty object
   options = utils.isPOJO(options) ? { ...options } : {};
   options._calledWithOptions = options._calledWithOptions || { ...options };
@@ -3823,7 +3829,9 @@ Document.prototype.$toObject = function(options, json) {
   }
 
   options.minimize = _minimize;
-  options._seen = options._seen || new Map();
+  if (!hasOnlyPrimitiveValues) {
+    options._seen = options._seen || new Map();
+  }
 
   const depopulate = options._calledWithOptions.depopulate
     ?? options._parentOptions?.depopulate
@@ -3854,7 +3862,14 @@ Document.prototype.$toObject = function(options, json) {
   // to save it from being overwritten by sub-transform functions
   // const originalTransform = options.transform;
 
-  let ret = clone(this._doc, options) || {};
+  let ret;
+  if (hasOnlyPrimitiveValues && !options.flattenObjectIds) {
+    // Fast path: if we don't have any nested objects or arrays, we only need a
+    // shallow clone.
+    ret = this.$__toObjectShallow();
+  } else {
+    ret = clone(this._doc, options) || {};
+  }
 
   options._skipSingleNestedGetters = true;
   const getters = options._calledWithOptions.getters
@@ -3912,6 +3927,26 @@ Document.prototype.$toObject = function(options, json) {
   return ret;
 };
 
+/*!
+ * Internal shallow clone alternative to `$toObject()`: much faster, no options processing
+ */
+
+Document.prototype.$__toObjectShallow = function $__toObjectShallow() {
+  const ret = {};
+  if (this._doc != null) {
+    for (const key of Object.keys(this._doc)) {
+      const value = this._doc[key];
+      if (value instanceof Date) {
+        ret[key] = new Date(value);
+      } else if (value !== undefined) {
+        ret[key] = value;
+      }
+    }
+  }
+
+  return ret;
+};
+
 /**
  * Converts this document into a plain-old JavaScript object ([POJO](https://masteringjs.io/tutorials/fundamentals/pojo)).
  *
@@ -5292,6 +5327,20 @@ Document.prototype.$clearModifiedPaths = function $clearModifiedPaths() {
   return this;
 };
 
+/*!
+ * Check if the given document only has primitive values
+ */
+
+Document.prototype.$__hasOnlyPrimitiveValues = function $__hasOnlyPrimitiveValues() {
+  return !this.$__.populated && !this.$__.wasPopulated && (this._doc == null || Object.values(this._doc).every(v => {
+    return v == null
+      || typeof v !== 'object'
+      || (utils.isNativeObject(v) && !Array.isArray(v))
+      || isBsonType(v, 'ObjectId')
+      || isBsonType(v, 'Decimal128');
+  }));
+};
+
 /*!
  * Module exports.
  */

diff --git a/lib/model.js b/lib/model.js
@@ -2854,16 +2854,19 @@ Model.$__insertMany = function(arr, options, callback) {
         // execute the callback synchronously
         return immediate(() => callback(null, doc));
       }
+      let createdNewDoc = false;
       if (!(doc instanceof _this)) {
         if (doc != null && typeof doc !== 'object') {
           return callback(new ObjectParameterError(doc, 'arr.' + index, 'insertMany'));
         }
         try {
           doc = new _this(doc);
+          createdNewDoc = true;
         } catch (err) {
           return callback(err);
         }
       }
+
       if (options.session != null) {
         doc.$session(options.session);
       }
@@ -2874,7 +2877,7 @@ Model.$__insertMany = function(arr, options, callback) {
         // execute the callback synchronously
         return immediate(() => callback(null, doc));
       }
-      doc.$validate().then(
+      doc.$validate(createdNewDoc ? { _skipParallelValidateCheck: true } : null).then(
         () => { callback(null, doc); },
         error => {
           if (ordered === false) {
@@ -2948,7 +2951,10 @@ Model.$__insertMany = function(arr, options, callback) {
       }
       const shouldSetTimestamps = (!options || options.timestamps !== false) && doc.initializeTimestamps && (!doc.$__ || doc.$__.timestamps !== false);
       if (shouldSetTimestamps) {
-        return doc.initializeTimestamps().toObject(internalToObjectOptions);
+        doc.initializeTimestamps();
+      }
+      if (doc.$__hasOnlyPrimitiveValues()) {
+        return doc.$__toObjectShallow();
       }
       return doc.toObject(internalToObjectOptions);
     });

diff --git a/test/document.unit.test.js b/test/document.unit.test.js
@@ -61,8 +61,6 @@ describe('toObject()', function() {
   it('doesnt crash with empty object (gh-3130)', function() {
     const d = new Stub();
     d._doc = undefined;
-    assert.doesNotThrow(function() {
-      d.toObject();
-    });
+    d.toObject();
   });
 });
diff --git a/test/model.populate.test.js b/test/model.populate.test.js
@@ -9449,7 +9449,6 @@ describe('model: populate:', function() {
         children: [{ type: 'ObjectId', ref: 'Child' }]
       }));
 
-
       const children = await Child.create([{ name: 'Luke' }, { name: 'Leia' }]);
 
       let doc = await Parent.create({ children, child: children[0] });
-Original file line number
+Diff line change
@@ Expand Up / @@ -9449,7 +9449,6 @@ describe('model: populate:', function() { @@
             children: [{ type: 'ObjectId', ref: 'Child' }]
           }));
           const children = await Child.create([{ name: 'Luke' }, { name: 'Leia' }]);
           let doc = await Parent.create({ children, child: children[0] });
@@ Expand Down @@