Skip to content

Commit ab9eb07

Browse files
notheotherbennotheotherben
committed
Updated Diff engine
We've moved to a new diff engine (codename Omnom) which is so much better than the original diff algorithm as to make any comparison laughable. The biggest reason for the move was to allow us to support array diffs in an intelligent way while still allowing single queries to function correctly. To achieve this, there are a few important implications which you need to take into account - namely that using the built in algorithm for combinations of pushes and pulls on arrays is a TERRIBLE IDEA. This is due to the way that arrays are handled to support single queries - since MongoDB doesn't allow combinations of $push/$pull/$set on an array within a single query for consistency reasons. USE IT FOR: - Changing the values of array elements (and their child values) without changing their position in the array - Adding array elements and making changes to existing ones (at the same time if you wish) BIG DON'TS: - Removing array elements while adding/modifying others - Insertion at the front of an array (Forces the array to be replaced)
1 parent d8134a9 commit ab9eb07

File tree

6 files changed

+527
-206
lines changed

6 files changed

+527
-206
lines changed

‎lib/Instance.js

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ var ObjectID = require('mongodb').ObjectID,
77
EventEmitter = require('events').EventEmitter,
88
debug = require('debug')('iridium:Instance'),
99

10-
validate = require('./utils/validation');
10+
validate = require('./utils/validation'),
11+
diff = require('./utils/diff');
1112

1213

1314
(require.modules || {}).Instance = module.exports = Instance;
@@ -301,37 +302,4 @@ Instance.forModel = function(model) {
301302
return ModelInstance;
302303
};
303304

304-
var diffPatch = Instance.diff = function (oldDoc, newDoc, path) {
305-
/// <signature>
306-
/// <summary>Creates a differential update query for use by MongoDB</summary>
307-
/// <param name="oldDoc" type="Object">The original document prior to any changes</param>
308-
/// <param name="newDoc" type="Object">The document containing the changes made to the original document</param>
309-
/// </signature>
310-
311-
"use strict";
312-
313-
var changes = {};
314-
315-
for (var k in newDoc) {
316-
if (Array.isArray(newDoc[k]) && Array.isArray(oldDoc[k])) {
317-
var different = newDoc.length !== oldDoc.length;
318-
for (var i = 0; i < newDoc[k].length && !different; i++) {
319-
if (oldDoc[k][i] !== newDoc[k][i]) different = true;
320-
}
321-
if (!different) continue;
322-
changes.$set = changes.$set || {};
323-
changes.$set[(path ? (path + '.') : '') + k] = newDoc[k];
324-
}
325-
else if (_.isPlainObject(newDoc[k]) && _.isPlainObject(oldDoc[k])) {
326-
// Make recursive diff update
327-
_.merge(changes, diffPatch(oldDoc[k], newDoc[k], (path ? (path + '.') : '') + k));
328-
}
329-
else {
330-
if (oldDoc[k] === newDoc[k]) continue;
331-
changes.$set = changes.$set || {};
332-
changes.$set[(path ? (path + '.') : '') + k] = newDoc[k];
333-
}
334-
}
335-
336-
return changes;
337-
};
305+
Instance.diff = diff;

‎lib/utils/diff.js

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
var _ = require('lodash');
2+
3+
module.exports = diff;
4+
5+
function diff(original, modified) {
6+
var omnom = new Omnom({
7+
cautious: true,
8+
orderedArrays: true
9+
});
10+
11+
omnom.diff(original, modified);
12+
13+
return omnom.changes;
14+
}
15+
16+
function Omnom(options) {
17+
this.options = options;
18+
this.changes = {};
19+
}
20+
21+
Omnom.prototype.diff = function(original, modified) {
22+
this.onObject(original, modified);
23+
};
24+
25+
Omnom.prototype.onObject = function(original, modified, changePath) {
26+
if(original === undefined || original === null)
27+
return (original !== modified) && this.set(changePath, modified);
28+
29+
if(typeof original == 'number' && typeof modified == 'number' && original !== modified)
30+
return this.inc(changePath, modified - original);
31+
32+
if(Array.isArray(original) && Array.isArray(modified))
33+
return this.onArray(original, modified, changePath);
34+
35+
if(!_.isPlainObject(original) || !_.isPlainObject(modified))
36+
return (original != modified) && this.set(changePath, modified);
37+
38+
_.each(modified, function(value, key) {
39+
// Handle array diffs in their own special way
40+
if(Array.isArray(value) && Array.isArray(original[key])) this.onArray(original[key], value, resolve(changePath, key));
41+
42+
// Otherwise, just keep going
43+
else this.onObject(original[key], value, resolve(changePath, key));
44+
}, this);
45+
46+
// Unset removed properties
47+
_.each(original, function(value, key) {
48+
if(modified[key] === undefined || modified[key] === null) return this.unset(resolve(changePath, key));
49+
}, this);
50+
};
51+
52+
Omnom.prototype.onArray = function(original, modified, changePath) {
53+
var i,j;
54+
55+
// Check if we can get from original => modified using just pulls
56+
if(original.length > modified.length) {
57+
var pulls = [];
58+
for(i = 0, j = 0; i < original.length && j < modified.length; i++) {
59+
if(almostEqual(original[i], modified[j])) j++;
60+
else pulls.push(original[i]);
61+
}
62+
63+
for(; i < original.length; i++)
64+
pulls.push(original[i]);
65+
66+
if(j === modified.length) {
67+
if(pulls.length === 1) return this.pull(changePath, pulls[0]);
68+
// We can complete using just pulls
69+
return this.pullAll(changePath, pulls);
70+
}
71+
72+
// If we have a smaller target array than our source, we will need to re-create it
73+
// regardless (if we want to do so in a single operation anyway)
74+
else return this.set(changePath, modified);
75+
}
76+
77+
// Check if we can get from original => modified using just pushes
78+
if(original.length < modified.length) {
79+
var canPush = true;
80+
for(i = 0; i < original.length; i++)
81+
if(almostEqual(original[i], modified[i]) < 1) {
82+
canPush = false;
83+
break;
84+
}
85+
86+
if(canPush) {
87+
for(i = original.length; i < modified.length; i++)
88+
this.push(changePath, modified[i]);
89+
return;
90+
}
91+
}
92+
93+
// Otherwise, we need to use $set to generate the new array
94+
95+
// Check how many manipulations would need to be performed, if it's more than half the array size
96+
// then rather re-create the array
97+
98+
var sets = [];
99+
var partials = [];
100+
for(i = 0; i < modified.length; i++) {
101+
var equality = almostEqual(original[i], modified[i]);
102+
if(equality === 0) sets.push(i);
103+
else if(equality < 1) partials.push(i);
104+
}
105+
106+
if(sets.length > modified.length / 2)
107+
return this.set(changePath, modified);
108+
109+
for(i = 0; i < sets.length; i++)
110+
this.set(resolve(changePath, sets[i].toString()), modified[sets[i]]);
111+
112+
for(i = 0; i < partials.length; i++)
113+
this.onObject(original[partials[i]], modified[partials[i]], resolve(changePath, partials[i].toString()));
114+
};
115+
116+
Omnom.prototype.set = function(path, value) {
117+
if(!this.changes.$set)
118+
this.changes.$set = {};
119+
120+
this.changes.$set[path] = value;
121+
};
122+
123+
Omnom.prototype.unset = function(path, value) {
124+
if(!this.changes.$unset)
125+
this.changes.$unset = {};
126+
127+
this.changes.$unset[path] = 1;
128+
};
129+
130+
Omnom.prototype.inc = function(path, value) {
131+
if(!this.changes.$inc)
132+
this.changes.$inc = {};
133+
134+
this.changes.$inc[path] = value;
135+
};
136+
137+
Omnom.prototype.push = function(path, value) {
138+
if(!this.changes.$push)
139+
this.changes.$push = {};
140+
141+
if(this.changes.$push[path]) {
142+
if(this.changes.$push[path].$each)
143+
this.changes.$push[path].$each.push(value);
144+
else
145+
this.changes.$push[path] = { $each: [this.changes.$push[path], value] };
146+
} else this.changes.$push[path] = value;
147+
};
148+
149+
Omnom.prototype.pull = function(path, value) {
150+
if(!this.changes.$pull)
151+
this.changes.$pull = {};
152+
153+
if(this.changes.$pullAll && this.changes.$pullAll[path]) {
154+
return this.changes.$pullAll[path].push(value);
155+
}
156+
157+
if(this.changes.$pull[path]) {
158+
this.pullAll(path, [this.changes.$pull[path], value]);
159+
delete this.changes.$pull[path];
160+
return;
161+
}
162+
163+
this.changes.$pull[path] = value;
164+
};
165+
166+
Omnom.prototype.pullAll = function(path, values) {
167+
if(!this.changes.$pullAll)
168+
this.changes.$pullAll = {};
169+
170+
this.changes.$pullAll[path] = values;
171+
};
172+
173+
function resolve() {
174+
var validArguments = [];
175+
Array.prototype.forEach.call(arguments, function(arg) {
176+
if(arg) validArguments.push(arg);
177+
});
178+
return validArguments.join('.');
179+
}
180+
181+
var almostEqual = function (o1, o2) {
182+
if(!_.isPlainObject(o1) || !_.isPlainObject(o2)) return o1 == o2 ? 1 : 0;
183+
184+
var o1i, o1k = Object.keys(o1);
185+
var o2i, o2k = Object.keys(o2);
186+
187+
var commonKeys = [];
188+
for(o1i = 0; o1i < o1k.length; o1i++)
189+
if(~o2k.indexOf(o1k[o1i])) commonKeys.push(o1k[o1i]);
190+
191+
var totalKeys = o1k.length + o2k.length - commonKeys.length;
192+
var keysDifference = totalKeys - commonKeys.length;
193+
194+
var requiredChanges = 0;
195+
for(var i = 0; i < commonKeys.length; i++)
196+
if(almostEqual(o1[commonKeys[i]], o2[commonKeys[i]]) < 1) requiredChanges++;
197+
198+
return 1 - (keysDifference / totalKeys) - (requiredChanges / commonKeys.length);
199+
};

‎test/diff.js

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
var diff = require('../lib/utils/diff');
2+
3+
describe('diff', function() {
4+
it('should correctly diff basic objects', function() {
5+
var o1 = {
6+
a: 1,
7+
b: 'test',
8+
c: 2,
9+
d: 'constant',
10+
e: 'old'
11+
};
12+
13+
var o2 = {
14+
a: 3,
15+
b: 'tested',
16+
c: 2,
17+
d: 'constant',
18+
f: 'new'
19+
};
20+
21+
var expected = {
22+
$inc: { a: 2 },
23+
$set: { b: 'tested', f: 'new' },
24+
$unset: { e: 1 }
25+
};
26+
27+
diff(o1, o2).should.eql(expected);
28+
});
29+
30+
it('should correctly diff complex objects', function() {
31+
var o1 = {
32+
a: { value: 1 },
33+
b: { value1: 1, value2: 1 },
34+
c: { value: 2 },
35+
d: { value: {} },
36+
e: { value: true }
37+
};
38+
39+
var o2 = {
40+
a: { value: 3 },
41+
b: { value1: 'tested', value2: 2 },
42+
c: { value: 2 },
43+
d: { value: {} },
44+
e: { value2: false }
45+
};
46+
47+
var expected = {
48+
$inc: { 'a.value': 2, 'b.value2': 1 },
49+
$set: { 'b.value1': 'tested', 'e.value2': false },
50+
$unset: { 'e.value': 1 }
51+
};
52+
53+
diff(o1, o2).should.eql(expected);
54+
});
55+
56+
describe('arrays', function() {
57+
it('should correctly handle arrays which can be pulled', function() {
58+
var a1 = { a: [1,2,3,4], b: [1,2,3,4] };
59+
var a2 = { a: [1,3,4], b: [1,3] };
60+
var expected = {
61+
$pull: { a: 2 },
62+
$pullAll: { b: [2,4] }
63+
};
64+
65+
diff(a1, a2).should.eql(expected);
66+
});
67+
68+
it('should correctly handle arrays which can be pushed', function() {
69+
var a1 = { a: [1,2,3,4], b: [1,2,3,4] };
70+
var a2 = { a: [1,2,3,4,5], b: [1,2,3,4,5,6] };
71+
var expected = {
72+
$push: { a: 5, b: { $each: [5,6] }}
73+
};
74+
75+
diff(a1, a2).should.eql(expected);
76+
});
77+
78+
it('should correctly handle arrays which should be replaced', function() {
79+
var a1 = { a: [1,2], b: [1,2,3] };
80+
var a2 = { a: [5,4,3], b: [5,4,3,2] };
81+
var expected = {
82+
$set: {
83+
a: [5,4,3],
84+
b: [5,4,3,2]
85+
}
86+
};
87+
88+
diff(a1, a2).should.eql(expected);
89+
});
90+
91+
it("should correctly handle arrays which can be partially modified", function() {
92+
var a1 = { a: [1,2,3,4], b: [1,2,3,4] };
93+
var a2 = { a: [1,2,5,4,5], b: [1,2,5,4,5,6] };
94+
var expected = {
95+
$set: {
96+
'a.2': 5,
97+
'a.4': 5,
98+
'b.2': 5,
99+
'b.4': 5,
100+
'b.5': 6
101+
}
102+
};
103+
104+
diff(a1, a2).should.eql(expected);
105+
});
106+
107+
it("should correctly diff array elements as objects", function() {
108+
var postDate = new Date();
109+
var a1 = { comments: [
110+
{ id: 1, title: 'Title 1', text: 'test text 1', posted: postDate },
111+
{ id: 2, title: 'Title 2', text: 'test text 2', posted: postDate },
112+
{ id: 3, title: 'Title 3', text: 'test text 3', posted: postDate }
113+
]};
114+
115+
var newDate = new Date(postDate.getTime() + 50);
116+
var a2 = { comments: [
117+
{ id: 1, title: 'Title 1', text: 'tested text 1', posted: postDate },
118+
{ id: 2, title: 'Title 2', text: 'tested text 2', posted: postDate },
119+
{ id: 3, title: 'Title 3', text: 'test text 3', posted: newDate }
120+
]};
121+
122+
var expected = {
123+
$set: {
124+
'comments.0.text': 'tested text 1',
125+
'comments.1.text': 'tested text 2',
126+
'comments.2.posted': newDate
127+
}
128+
};
129+
130+
diff(a1, a2).should.eql(expected);
131+
});
132+
});
133+
});

0 commit comments

Comments
 (0)