Skip to content

Commit d30a3b9

Browse files
vkarpov15vkarpov15
authored
Merge pull request #14724 from Automattic/vkarpov15/gh-14719
Performance improvements for `insertMany()`
2 parents 7d742e2 + 8223f91 commit d30a3b9

File tree

5 files changed

+130
-40
lines changed

5 files changed

+130
-40
lines changed

‎benchmarks/insertManySimple.js

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
'use strict';
2+
3+
const mongoose = require('../');
4+
5+
run().catch(err => {
6+
console.error(err);
7+
process.exit(-1);
8+
});
9+
10+
async function run() {
11+
await mongoose.connect('mongodb://127.0.0.1:27017/mongoose_benchmark');
12+
const FooSchema = new mongoose.Schema({ foo: String });
13+
const FooModel = mongoose.model('Foo', FooSchema);
14+
15+
if (!process.env.MONGOOSE_BENCHMARK_SKIP_SETUP) {
16+
await FooModel.deleteMany({});
17+
}
18+
19+
const numDocs = 1500;
20+
const docs = [];
21+
for (let i = 0; i < numDocs; ++i) {
22+
docs.push({ foo: 'test foo ' + i });
23+
}
24+
25+
const numIterations = 200;
26+
const insertStart = Date.now();
27+
for (let i = 0; i < numIterations; ++i) {
28+
await FooModel.insertMany(docs);
29+
}
30+
const insertEnd = Date.now();
31+
32+
const results = {
33+
'Average insertMany time ms': +((insertEnd - insertStart) / numIterations).toFixed(2)
34+
};
35+
36+
console.log(JSON.stringify(results, null, ' '));
37+
process.exit(0);
38+
}

‎lib/document.js

+83-34
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ const getKeysInSchemaOrder = require('./helpers/schema/getKeysInSchemaOrder');
3131
const getSubdocumentStrictValue = require('./helpers/schema/getSubdocumentStrictValue');
3232
const handleSpreadDoc = require('./helpers/document/handleSpreadDoc');
3333
const immediate = require('./helpers/immediate');
34+
const isBsonType = require('./helpers/isBsonType');
3435
const isDefiningProjection = require('./helpers/projection/isDefiningProjection');
3536
const isExclusive = require('./helpers/projection/isExclusive');
3637
const isPathExcluded = require('./helpers/projection/isPathExcluded');
@@ -2611,17 +2612,6 @@ Document.prototype.validate = async function validate(pathsToValidate, options)
26112612
let parallelValidate;
26122613
this.$op = 'validate';
26132614

2614-
if (this.$isSubdocument != null) {
2615-
// Skip parallel validate check for subdocuments
2616-
} else if (this.$__.validating) {
2617-
parallelValidate = new ParallelValidateError(this, {
2618-
parentStack: options && options.parentStack,
2619-
conflictStack: this.$__.validating.stack
2620-
});
2621-
} else {
2622-
this.$__.validating = new ParallelValidateError(this, { parentStack: options && options.parentStack });
2623-
}
2624-
26252615
if (arguments.length === 1) {
26262616
if (typeof arguments[0] === 'object' && !Array.isArray(arguments[0])) {
26272617
options = arguments[0];
@@ -2632,6 +2622,18 @@ Document.prototype.validate = async function validate(pathsToValidate, options)
26322622
const isOnePathOnly = options.pathsToSkip.indexOf(' ') === -1;
26332623
options.pathsToSkip = isOnePathOnly ? [options.pathsToSkip] : options.pathsToSkip.split(' ');
26342624
}
2625+
const _skipParallelValidateCheck = options && options._skipParallelValidateCheck;
2626+
2627+
if (this.$isSubdocument != null) {
2628+
// Skip parallel validate check for subdocuments
2629+
} else if (this.$__.validating && !_skipParallelValidateCheck) {
2630+
parallelValidate = new ParallelValidateError(this, {
2631+
parentStack: options && options.parentStack,
2632+
conflictStack: this.$__.validating.stack
2633+
});
2634+
} else if (!_skipParallelValidateCheck) {
2635+
this.$__.validating = new ParallelValidateError(this, { parentStack: options && options.parentStack });
2636+
}
26352637

26362638
if (parallelValidate != null) {
26372639
throw parallelValidate;
@@ -3480,31 +3482,33 @@ Document.prototype.$__reset = function reset() {
34803482
let _this = this;
34813483

34823484
// Skip for subdocuments
3483-
const subdocs = this.$parent() === this ? this.$getAllSubdocs() : [];
3484-
const resetArrays = new Set();
3485-
for (const subdoc of subdocs) {
3486-
const fullPathWithIndexes = subdoc.$__fullPathWithIndexes();
3487-
subdoc.$__reset();
3488-
if (this.isModified(fullPathWithIndexes) || isParentInit(fullPathWithIndexes)) {
3489-
if (subdoc.$isDocumentArrayElement) {
3490-
resetArrays.add(subdoc.parentArray());
3491-
} else {
3492-
const parent = subdoc.$parent();
3493-
if (parent === this) {
3494-
this.$__.activePaths.clearPath(subdoc.$basePath);
3495-
} else if (parent != null && parent.$isSubdocument) {
3496-
// If map path underneath subdocument, may end up with a case where
3497-
// map path is modified but parent still needs to be reset. See gh-10295
3498-
parent.$__reset();
3485+
const subdocs = !this.$isSubdocument ? this.$getAllSubdocs() : null;
3486+
if (subdocs && subdocs.length > 0) {
3487+
const resetArrays = new Set();
3488+
for (const subdoc of subdocs) {
3489+
const fullPathWithIndexes = subdoc.$__fullPathWithIndexes();
3490+
subdoc.$__reset();
3491+
if (this.isModified(fullPathWithIndexes) || isParentInit(fullPathWithIndexes)) {
3492+
if (subdoc.$isDocumentArrayElement) {
3493+
resetArrays.add(subdoc.parentArray());
3494+
} else {
3495+
const parent = subdoc.$parent();
3496+
if (parent === this) {
3497+
this.$__.activePaths.clearPath(subdoc.$basePath);
3498+
} else if (parent != null && parent.$isSubdocument) {
3499+
// If map path underneath subdocument, may end up with a case where
3500+
// map path is modified but parent still needs to be reset. See gh-10295
3501+
parent.$__reset();
3502+
}
34993503
}
35003504
}
35013505
}
3502-
}
35033506

3504-
for (const array of resetArrays) {
3505-
this.$__.activePaths.clearPath(array.$path());
3506-
array[arrayAtomicsBackupSymbol] = array[arrayAtomicsSymbol];
3507-
array[arrayAtomicsSymbol] = {};
3507+
for (const array of resetArrays) {
3508+
this.$__.activePaths.clearPath(array.$path());
3509+
array[arrayAtomicsBackupSymbol] = array[arrayAtomicsSymbol];
3510+
array[arrayAtomicsSymbol] = {};
3511+
}
35083512
}
35093513

35103514
function isParentInit(path) {
@@ -3809,6 +3813,8 @@ Document.prototype.$__handleReject = function handleReject(err) {
38093813
Document.prototype.$toObject = function(options, json) {
38103814
const defaultOptions = this.$__schema._defaultToObjectOptions(json);
38113815

3816+
const hasOnlyPrimitiveValues = this.$__hasOnlyPrimitiveValues();
3817+
38123818
// If options do not exist or is not an object, set it to empty object
38133819
options = utils.isPOJO(options) ? { ...options } : {};
38143820
options._calledWithOptions = options._calledWithOptions || { ...options };
@@ -3823,7 +3829,9 @@ Document.prototype.$toObject = function(options, json) {
38233829
}
38243830

38253831
options.minimize = _minimize;
3826-
options._seen = options._seen || new Map();
3832+
if (!hasOnlyPrimitiveValues) {
3833+
options._seen = options._seen || new Map();
3834+
}
38273835

38283836
const depopulate = options._calledWithOptions.depopulate
38293837
?? options._parentOptions?.depopulate
@@ -3854,7 +3862,14 @@ Document.prototype.$toObject = function(options, json) {
38543862
// to save it from being overwritten by sub-transform functions
38553863
// const originalTransform = options.transform;
38563864

3857-
let ret = clone(this._doc, options) || {};
3865+
let ret;
3866+
if (hasOnlyPrimitiveValues && !options.flattenObjectIds) {
3867+
// Fast path: if we don't have any nested objects or arrays, we only need a
3868+
// shallow clone.
3869+
ret = this.$__toObjectShallow();
3870+
} else {
3871+
ret = clone(this._doc, options) || {};
3872+
}
38583873

38593874
options._skipSingleNestedGetters = true;
38603875
const getters = options._calledWithOptions.getters
@@ -3912,6 +3927,26 @@ Document.prototype.$toObject = function(options, json) {
39123927
return ret;
39133928
};
39143929

3930+
/*!
3931+
* Internal shallow clone alternative to `$toObject()`: much faster, no options processing
3932+
*/
3933+
3934+
Document.prototype.$__toObjectShallow = function $__toObjectShallow() {
3935+
const ret = {};
3936+
if (this._doc != null) {
3937+
for (const key of Object.keys(this._doc)) {
3938+
const value = this._doc[key];
3939+
if (value instanceof Date) {
3940+
ret[key] = new Date(value);
3941+
} else if (value !== undefined) {
3942+
ret[key] = value;
3943+
}
3944+
}
3945+
}
3946+
3947+
return ret;
3948+
};
3949+
39153950
/**
39163951
* Converts this document into a plain-old JavaScript object ([POJO](https://masteringjs.io/tutorials/fundamentals/pojo)).
39173952
*
@@ -5292,6 +5327,20 @@ Document.prototype.$clearModifiedPaths = function $clearModifiedPaths() {
52925327
return this;
52935328
};
52945329

5330+
/*!
5331+
* Check if the given document only has primitive values
5332+
*/
5333+
5334+
Document.prototype.$__hasOnlyPrimitiveValues = function $__hasOnlyPrimitiveValues() {
5335+
return !this.$__.populated && !this.$__.wasPopulated && (this._doc == null || Object.values(this._doc).every(v => {
5336+
return v == null
5337+
|| typeof v !== 'object'
5338+
|| (utils.isNativeObject(v) && !Array.isArray(v))
5339+
|| isBsonType(v, 'ObjectId')
5340+
|| isBsonType(v, 'Decimal128');
5341+
}));
5342+
};
5343+
52955344
/*!
52965345
* Module exports.
52975346
*/

‎lib/model.js

+8-2
Original file line numberDiff line numberDiff line change
@@ -2854,16 +2854,19 @@ Model.$__insertMany = function(arr, options, callback) {
28542854
// execute the callback synchronously
28552855
return immediate(() => callback(null, doc));
28562856
}
2857+
let createdNewDoc = false;
28572858
if (!(doc instanceof _this)) {
28582859
if (doc != null && typeof doc !== 'object') {
28592860
return callback(new ObjectParameterError(doc, 'arr.' + index, 'insertMany'));
28602861
}
28612862
try {
28622863
doc = new _this(doc);
2864+
createdNewDoc = true;
28632865
} catch (err) {
28642866
return callback(err);
28652867
}
28662868
}
2869+
28672870
if (options.session != null) {
28682871
doc.$session(options.session);
28692872
}
@@ -2874,7 +2877,7 @@ Model.$__insertMany = function(arr, options, callback) {
28742877
// execute the callback synchronously
28752878
return immediate(() => callback(null, doc));
28762879
}
2877-
doc.$validate().then(
2880+
doc.$validate(createdNewDoc ? { _skipParallelValidateCheck: true } : null).then(
28782881
() => { callback(null, doc); },
28792882
error => {
28802883
if (ordered === false) {
@@ -2948,7 +2951,10 @@ Model.$__insertMany = function(arr, options, callback) {
29482951
}
29492952
const shouldSetTimestamps = (!options || options.timestamps !== false) && doc.initializeTimestamps && (!doc.$__ || doc.$__.timestamps !== false);
29502953
if (shouldSetTimestamps) {
2951-
return doc.initializeTimestamps().toObject(internalToObjectOptions);
2954+
doc.initializeTimestamps();
2955+
}
2956+
if (doc.$__hasOnlyPrimitiveValues()) {
2957+
return doc.$__toObjectShallow();
29522958
}
29532959
return doc.toObject(internalToObjectOptions);
29542960
});

‎test/document.unit.test.js

+1-3
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,6 @@ describe('toObject()', function() {
6161
it('doesnt crash with empty object (gh-3130)', function() {
6262
const d = new Stub();
6363
d._doc = undefined;
64-
assert.doesNotThrow(function() {
65-
d.toObject();
66-
});
64+
d.toObject();
6765
});
6866
});

‎test/model.populate.test.js

-1
Original file line numberDiff line numberDiff line change
@@ -9449,7 +9449,6 @@ describe('model: populate:', function() {
94499449
children: [{ type: 'ObjectId', ref: 'Child' }]
94509450
}));
94519451

9452-
94539452
const children = await Child.create([{ name: 'Luke' }, { name: 'Leia' }]);
94549453

94559454
let doc = await Parent.create({ children, child: children[0] });

0 commit comments

Comments
 (0)