From bb011df65049a7915140e7157fa303777e91c4e8 Mon Sep 17 00:00:00 2001 From: obeliskos Date: Tue, 13 Feb 2018 17:15:54 -0500 Subject: [PATCH] cleanup of simplesort, added js comparisons, updated wiki md - cleaned up simplesort logic - added simplified javascript comparisons $jgt, $jgte, $jlt, $jlte described in Query Examples wiki page - updated tutorials folder with markdown versions of several modified wiki pages - rebuilt minified --- build/lokijs.min.js | 4 +- src/lokijs.js | 122 ++++++++++++-------- tutorials/Indexing and Query performance.md | 60 +++++++--- tutorials/Query Examples.md | 24 ++++ 4 files changed, 150 insertions(+), 60 deletions(-) diff --git a/build/lokijs.min.js b/build/lokijs.min.js index 86079250..fae236ba 100644 --- a/build/lokijs.min.js +++ b/build/lokijs.min.js @@ -1,3 +1,3 @@ -(function(root,factory){if(typeof define==="function"&&define.amd){define([],factory)}else if(typeof exports==="object"){module.exports=factory()}else{root.loki=factory()}})(this,function(){return function(){"use strict";var hasOwnProperty=Object.prototype.hasOwnProperty;var Utils={copyProperties:function(src,dest){var prop;for(prop in src){dest[prop]=src[prop]}},resolveTransformObject:function(subObj,params,depth){var prop,pname;if(typeof depth!=="number"){depth=0}if(++depth>=10)return subObj;for(prop in subObj){if(typeof subObj[prop]==="string"&&subObj[prop].indexOf("[%lktxp]")===0){pname=subObj[prop].substring(8);if(params.hasOwnProperty(pname)){subObj[prop]=params[pname]}}else if(typeof subObj[prop]==="object"){subObj[prop]=Utils.resolveTransformObject(subObj[prop],params,depth)}}return subObj},resolveTransformParams:function(transform,params){var idx,clonedStep,resolvedTransform=[];if(typeof params==="undefined")return transform;for(idx=0;idxcv2)return false;return equal}if(cv1===cv1&&cv2!==cv2){return true}if(cv2===cv2&&cv1!==cv1){return false}if(prop1prop2)return false;if(prop1==prop2)return equal;cv1=prop1.toString();cv2=prop2.toString();if(cv1t2}}cv1=Number(prop1);cv2=Number(prop2);if(cv1===cv1&&cv2===cv2){if(cv1>cv2)return true;if(cv1prop2)return true;if(prop1cv2){return true}if(cv1==cv2){return equal}return false}function sortHelper(prop1,prop2,desc){if(aeqHelper(prop1,prop2))return 0;if(ltHelper(prop1,prop2,false)){return desc?1:-1}if(gtHelper(prop1,prop2,false)){return desc?-1:1}return 0}function compoundeval(properties,obj1,obj2){var res=0;var prop,field,val1,val2,arr;for(var i=0,len=properties.length;i=paths.length){valueFound=fun(element,value)}else if(Array.isArray(element)){for(var index=0,len=element.length;index0){throw new Error("disableMeta option cannot be passed as true when ttl is enabled")}}for(i=0;i=0){return this.serializeCollection({delimited:options.delimited,delimiter:options.delimiter,collectionIndex:options.partition})}dbcopy=new Loki(this.filename);dbcopy.loadJSONObject(this);for(idx=0;idxcollCount){done=true}}else{currObject=JSON.parse(workarray[lineIndex]);cdb.collections[collIndex].data.push(currObject)}workarray[lineIndex++]=null}return cdb};Loki.prototype.deserializeCollection=function(destructuredSource,options){var workarray=[];var idx,len;options=options||{};if(!options.hasOwnProperty("partitioned")){options.partitioned=false}if(!options.hasOwnProperty("delimited")){options.delimited=true}if(!options.hasOwnProperty("delimiter")){options.delimiter=this.options.destructureDelimiter}if(options.delimited){workarray=destructuredSource.split(options.delimiter);workarray.pop()}else{workarray=destructuredSource}len=workarray.length;for(idx=0;idx=cdlen)doneWithPartition=true}if(pageLen>=this.options.pageSize)doneWithPage=true;if(!doneWithPage||doneWithPartition){pageBuilder+=this.options.delimiter;pageLen+=delimlen}if(doneWithPartition||doneWithPage){this.adapter.saveDatabase(keyname,pageBuilder,pageSaveCallback);return}}};function LokiFsAdapter(){this.fs=require("fs")}LokiFsAdapter.prototype.loadDatabase=function loadDatabase(dbname,callback){var self=this;this.fs.stat(dbname,function(err,stats){if(!err&&stats.isFile()){self.fs.readFile(dbname,{encoding:"utf8"},function readFileCallback(err,data){if(err){callback(new Error(err))}else{callback(data)}})}else{callback(null)}})};LokiFsAdapter.prototype.saveDatabase=function saveDatabase(dbname,dbstring,callback){var self=this;var tmpdbname=dbname+"~";this.fs.writeFile(tmpdbname,dbstring,function writeFileCallback(err){if(err){callback(new Error(err))}else{self.fs.rename(tmpdbname,dbname,callback)}})};LokiFsAdapter.prototype.deleteDatabase=function deleteDatabase(dbname,callback){this.fs.unlink(dbname,function deleteDatabaseCallback(err){if(err){callback(new Error(err))}else{callback()}})};function LokiLocalStorageAdapter(){}LokiLocalStorageAdapter.prototype.loadDatabase=function loadDatabase(dbname,callback){if(localStorageAvailable()){callback(localStorage.getItem(dbname))}else{callback(new Error("localStorage is not available"))}};LokiLocalStorageAdapter.prototype.saveDatabase=function saveDatabase(dbname,dbstring,callback){if(localStorageAvailable()){localStorage.setItem(dbname,dbstring);callback(null)}else{callback(new Error("localStorage is not available"))}};LokiLocalStorageAdapter.prototype.deleteDatabase=function deleteDatabase(dbname,callback){if(localStorageAvailable()){localStorage.removeItem(dbname);callback(null)}else{callback(new Error("localStorage is not available"))}};Loki.prototype.throttledSaveDrain=function(callback,options){var self=this;var now=(new Date).getTime();if(!this.throttledSaves){callback(true)}options=options||{};if(!options.hasOwnProperty("recursiveWait")){options.recursiveWait=true} -if(!options.hasOwnProperty("recursiveWaitLimit")){options.recursiveWaitLimit=false}if(!options.hasOwnProperty("recursiveWaitLimitDuration")){options.recursiveWaitLimitDuration=2e3}if(!options.hasOwnProperty("started")){options.started=(new Date).getTime()}if(this.throttledSaves&&this.throttledSavePending){if(options.recursiveWait){this.throttledCallbacks.push(function(){if(self.throttledSavePending){if(options.recursiveWaitLimit&&now-options.started>options.recursiveWaitLimitDuration){callback(false);return}self.throttledSaveDrain(callback,options);return}else{callback(true);return}})}else{this.throttledCallbacks.push(callback);return}}else{callback(true)}};Loki.prototype.loadDatabaseInternal=function(options,callback){var cFun=callback||function(err,data){if(err){throw err}},self=this;if(this.persistenceAdapter!==null){this.persistenceAdapter.loadDatabase(this.filename,function loadDatabaseCallback(dbString){if(typeof dbString==="string"){var parseSuccess=false;try{self.loadJSON(dbString,options||{});parseSuccess=true}catch(err){cFun(err)}if(parseSuccess){cFun(null);self.emit("loaded","database "+self.filename+" loaded")}}else{if(!dbString){cFun(null);self.emit("loaded","empty database "+self.filename+" loaded");return}if(dbString instanceof Error){cFun(dbString);return}if(typeof dbString==="object"){self.loadJSONObject(dbString,options||{});cFun(null);self.emit("loaded","database "+self.filename+" loaded");return}cFun("unexpected adapter response : "+dbString)}})}else{cFun(new Error("persistenceAdapter not configured"))}};Loki.prototype.loadDatabase=function(options,callback){var self=this;if(!this.throttledSaves){this.loadDatabaseInternal(options,callback);return}this.throttledSaveDrain(function(success){if(success){self.throttledSavePending=true;self.loadDatabaseInternal(options,function(err){if(self.throttledCallbacks.length===0){self.throttledSavePending=false}else{self.saveDatabase()}if(typeof callback==="function"){callback(err)}});return}else{if(typeof callback==="function"){callback(new Error("Unable to pause save throttling long enough to read database"))}}},options)};Loki.prototype.saveDatabaseInternal=function(callback){var cFun=callback||function(err){if(err){throw err}return},self=this;if(this.persistenceAdapter!==null){if(this.persistenceAdapter.mode==="reference"&&typeof this.persistenceAdapter.exportDatabase==="function"){this.persistenceAdapter.exportDatabase(this.filename,this.copy({removeNonSerializable:true}),function exportDatabaseCallback(err){self.autosaveClearFlags();cFun(err)})}else{self.autosaveClearFlags();this.persistenceAdapter.saveDatabase(this.filename,self.serialize(),function saveDatabasecallback(err){cFun(err)})}}else{cFun(new Error("persistenceAdapter not configured"))}};Loki.prototype.saveDatabase=function(callback){if(!this.throttledSaves){this.saveDatabaseInternal(callback);return}if(this.throttledSavePending){this.throttledCallbacks.push(callback);return}var localCallbacks=this.throttledCallbacks;this.throttledCallbacks=[];localCallbacks.unshift(callback);this.throttledSavePending=true;var self=this;this.saveDatabaseInternal(function(err){self.throttledSavePending=false;localCallbacks.forEach(function(pcb){if(typeof pcb==="function"){setTimeout(function(){pcb(err)},1)}});if(self.throttledCallbacks.length>0){self.saveDatabase()}})};Loki.prototype.save=Loki.prototype.saveDatabase;Loki.prototype.deleteDatabase=function(options,callback){var cFun=callback||function(err,data){if(err){throw err}};if(typeof options==="function"&&!callback){cFun=options}if(this.persistenceAdapter!==null){this.persistenceAdapter.deleteDatabase(this.filename,function deleteDatabaseCallback(err){cFun(err)})}else{cFun(new Error("persistenceAdapter not configured"))}};Loki.prototype.autosaveDirty=function(){for(var idx=0;idx0){this.filteredrows=[]}this.filterInitialized=false;return this};Resultset.prototype.toJSON=function(){var copy=this.copy();copy.collection=null;return copy};Resultset.prototype.limit=function(qty){if(!this.filterInitialized&&this.filteredrows.length===0){this.filteredrows=this.collection.prepareFullDocIndex()}var rscopy=new Resultset(this.collection);rscopy.filteredrows=this.filteredrows.slice(0,qty);rscopy.filterInitialized=true;return rscopy};Resultset.prototype.offset=function(pos){if(!this.filterInitialized&&this.filteredrows.length===0){this.filteredrows=this.collection.prepareFullDocIndex()}var rscopy=new Resultset(this.collection);rscopy.filteredrows=this.filteredrows.slice(pos);rscopy.filterInitialized=true;return rscopy};Resultset.prototype.copy=function(){var result=new Resultset(this.collection);if(this.filteredrows.length>0){result.filteredrows=this.filteredrows.slice()}result.filterInitialized=this.filterInitialized;return result};Resultset.prototype.branch=Resultset.prototype.copy;Resultset.prototype.transform=function(transform,parameters){var idx,step,rs=this;if(typeof transform==="string"){if(this.collection.transforms.hasOwnProperty(transform)){transform=this.collection.transforms[transform]}}if(typeof transform!=="object"||!Array.isArray(transform)){throw new Error("Invalid transform")}if(typeof parameters!=="undefined"){transform=Utils.resolveTransformParams(transform,parameters)}for(idx=0;idxobj2[propname])return 1;if(obj1[propname]1){return this.find({$and:filters},firstOnly)}}if(!property||queryObject==="getAll"){if(firstOnly){this.filteredrows=this.collection.data.length>0?[0]:[];this.filterInitialized=true}return this}if(property==="$and"||property==="$or"){this[property](queryObjectOp);if(firstOnly&&this.filteredrows.length>1){this.filteredrows=this.filteredrows.slice(0,1)}return this}if(queryObjectOp===null||(typeof queryObjectOp!=="object"||queryObjectOp instanceof Date)){operator="$eq";value=queryObjectOp}else if(typeof queryObjectOp==="object"){for(key in queryObjectOp){if(hasOwnProperty.call(queryObjectOp,key)){operator=key;value=queryObjectOp[key];break}}}else{throw new Error("Do not know what you want to do.")}if(operator==="$regex"){if(Array.isArray(value)){value=new RegExp(value[0],value[1])}else if(!(value instanceof RegExp)){value=new RegExp(value)}}var usingDotNotation=property.indexOf(".")!==-1;var doIndexCheck=!usingDotNotation&&!this.filterInitialized;if(doIndexCheck&&this.collection.binaryIndices[property]&&indexedOps[operator]){if(this.collection.adaptiveBinaryIndices!==true){this.collection.ensureIndex(property)}searchByIndex=true;index=this.collection.binaryIndices[property]}var fun=LokiOps[operator];var t=this.collection.data;var i=0,len=0;var filter,rowIdx=0;if(this.filterInitialized){filter=this.filteredrows;len=filter.length;if(usingDotNotation){property=property.split(".");for(i=0;i=0){this.filterPipeline[idx]=filter;return this.reapplyFilters()}this.cachedresultset=null;if(this.options.persistent){this.resultdata=[];this.resultsdirty=true}this._addFilter(filter);if(this.sortFunction||this.sortCriteria){this.queueSortPhase()}else{this.queueRebuildEvent()}return this};DynamicView.prototype.applyFind=function(query,uid){this.applyFilter({type:"find",val:query,uid:uid});return this};DynamicView.prototype.applyWhere=function(fun,uid){this.applyFilter({type:"where",val:fun,uid:uid});return this};DynamicView.prototype.removeFilter=function(uid){var idx=this._indexOfFilterWithId(uid);if(idx<0){throw new Error("Dynamic view does not contain a filter with ID: "+uid)}this.filterPipeline.splice(idx,1);this.reapplyFilters();return this};DynamicView.prototype.count=function(){if(this.resultsdirty){this.resultdata=this.resultset.data()}return this.resultset.count()};DynamicView.prototype.data=function(options){if(this.sortDirty||this.resultsdirty){this.performSortPhase({suppressRebuildEvent:true})}return this.options.persistent?this.resultdata:this.resultset.data(options)};DynamicView.prototype.queueRebuildEvent=function(){if(this.rebuildPending){return}this.rebuildPending=true;var self=this;setTimeout(function(){if(self.rebuildPending){self.rebuildPending=false;self.emit("rebuild",self)}},this.options.minRebuildInterval)};DynamicView.prototype.queueSortPhase=function(){if(this.sortDirty){return}this.sortDirty=true;var self=this;if(this.options.sortPriority==="active"){setTimeout(function(){self.performSortPhase()},this.options.minRebuildInterval)}else{this.queueRebuildEvent()}};DynamicView.prototype.performSortPhase=function(options){if(!this.sortDirty&&!this.resultsdirty){return}options=options||{};if(this.sortDirty){if(this.sortFunction){this.resultset.sort(this.sortFunction)}else if(this.sortCriteria){this.resultset.compoundsort(this.sortCriteria)}this.sortDirty=false}if(this.options.persistent){this.resultdata=this.resultset.data();this.resultsdirty=false}if(!options.suppressRebuildEvent){this.emit("rebuild",this)}};DynamicView.prototype.evaluateDocument=function(objIndex,isNew){if(!this.resultset.filterInitialized){if(this.options.persistent){this.resultdata=this.resultset.data()}if(this.sortFunction||this.sortCriteria){this.queueSortPhase()}else{this.queueRebuildEvent()}return}var ofr=this.resultset.filteredrows;var oldPos=isNew?-1:ofr.indexOf(+objIndex);var oldlen=ofr.length;var evalResultset=new Resultset(this.collection);evalResultset.filteredrows=[objIndex];evalResultset.filterInitialized=true;var filter;for(var idx=0,len=this.filterPipeline.length;idxobjIndex){ofr[idx]--}}};DynamicView.prototype.mapReduce=function(mapFunction,reduceFunction){try{return reduceFunction(this.data().map(mapFunction))}catch(err){throw err}};function Collection(name,options){this.name=name;this.data=[];this.idIndex=[];this.binaryIndices={};this.constraints={unique:{},exact:{}};this.uniqueNames=[];this.transforms={};this.objType=name;this.dirty=true;this.cachedIndex=null;this.cachedBinaryIndex=null;this.cachedData=null;var self=this;options=options||{};if(options.hasOwnProperty("unique")){if(!Array.isArray(options.unique)){options.unique=[options.unique]}options.unique.forEach(function(prop){self.uniqueNames.push(prop);self.constraints.unique[prop]=new UniqueIndex(prop)})}if(options.hasOwnProperty("exact")){options.exact.forEach(function(prop){self.constraints.exact[prop]=new ExactIndex(prop)})}this.adaptiveBinaryIndices=options.hasOwnProperty("adaptiveBinaryIndices")?options.adaptiveBinaryIndices:true;this.transactional=options.hasOwnProperty("transactional")?options.transactional:false;this.cloneObjects=options.hasOwnProperty("clone")?options.clone:false;this.cloneMethod=options.hasOwnProperty("cloneMethod")?options.cloneMethod:"parse-stringify";this.asyncListeners=options.hasOwnProperty("asyncListeners")?options.asyncListeners:false;this.disableMeta=options.hasOwnProperty("disableMeta")?options.disableMeta:false;this.disableChangesApi=options.hasOwnProperty("disableChangesApi")?options.disableChangesApi:true;this.disableDeltaChangesApi=options.hasOwnProperty("disableDeltaChangesApi")?options.disableDeltaChangesApi:true;if(this.disableChangesApi){this.disableDeltaChangesApi=true}this.autoupdate=options.hasOwnProperty("autoupdate")?options.autoupdate:false;this.serializableIndices=options.hasOwnProperty("serializableIndices")?options.serializableIndices:true;this.ttl={age:null,ttlInterval:null,daemon:null};this.setTTL(options.ttl||-1,options.ttlInterval);this.maxId=0;this.DynamicViews=[];this.events={insert:[],update:[],"pre-insert":[],"pre-update":[],close:[],flushbuffer:[],error:[],delete:[],warning:[]};this.changes=[];this.ensureId();var indices=[];if(options&&options.indices){if(Object.prototype.toString.call(options.indices)==="[object Array]"){indices=options.indices}else if(typeof options.indices==="string"){indices=[options.indices]}else{throw new TypeError("Indices needs to be a string or an array of strings")}}for(var idx=0;idx=0||propertyName=="$loki"||propertyName=="meta"){delta[propertyName]=newObject[propertyName]}else{var propertyDelta=getObjectDelta(oldObject[propertyName],newObject[propertyName]);if(typeof propertyDelta!=="undefined"&&propertyDelta!={}){delta[propertyName]=propertyDelta}}}}return Object.keys(delta).length===0?undefined:delta}else{return oldObject===newObject?undefined:newObject}}this.getObjectDelta=getObjectDelta;function flushChanges(){self.changes=[]}this.getChanges=function(){return self.changes};this.flushChanges=flushChanges;function insertMeta(obj){var len,idx;if(self.disableMeta||!obj){return}if(Array.isArray(obj)){len=obj.length;for(idx=0;idx=10)return subObj;for(prop in subObj){if(typeof subObj[prop]==="string"&&subObj[prop].indexOf("[%lktxp]")===0){pname=subObj[prop].substring(8);if(params.hasOwnProperty(pname)){subObj[prop]=params[pname]}}else if(typeof subObj[prop]==="object"){subObj[prop]=Utils.resolveTransformObject(subObj[prop],params,depth)}}return subObj},resolveTransformParams:function(transform,params){var idx,clonedStep,resolvedTransform=[];if(typeof params==="undefined")return transform;for(idx=0;idxcv2)return false;return equal}if(cv1===cv1&&cv2!==cv2){return true}if(cv2===cv2&&cv1!==cv1){return false}if(prop1prop2)return false;if(prop1==prop2)return equal;cv1=prop1.toString();cv2=prop2.toString();if(cv1t2}}cv1=Number(prop1);cv2=Number(prop2);if(cv1===cv1&&cv2===cv2){if(cv1>cv2)return true;if(cv1prop2)return true;if(prop1cv2){return true}if(cv1==cv2){return equal}return false}function sortHelper(prop1,prop2,desc){if(aeqHelper(prop1,prop2))return 0;if(ltHelper(prop1,prop2,false)){return desc?1:-1}if(gtHelper(prop1,prop2,false)){return desc?-1:1}return 0}function compoundeval(properties,obj1,obj2){var res=0;var prop,field,val1,val2,arr;for(var i=0,len=properties.length;i=paths.length){valueFound=fun(element,value)}else if(Array.isArray(element)){for(var index=0,len=element.length;indexb},$jgte:function(a,b){return a>=b},$jlt:function(a,b){return a0){throw new Error("disableMeta option cannot be passed as true when ttl is enabled")}}for(i=0;i=0){return this.serializeCollection({delimited:options.delimited,delimiter:options.delimiter,collectionIndex:options.partition})}dbcopy=new Loki(this.filename);dbcopy.loadJSONObject(this);for(idx=0;idxcollCount){done=true}}else{currObject=JSON.parse(workarray[lineIndex]);cdb.collections[collIndex].data.push(currObject)}workarray[lineIndex++]=null}return cdb};Loki.prototype.deserializeCollection=function(destructuredSource,options){var workarray=[];var idx,len;options=options||{};if(!options.hasOwnProperty("partitioned")){options.partitioned=false}if(!options.hasOwnProperty("delimited")){options.delimited=true}if(!options.hasOwnProperty("delimiter")){options.delimiter=this.options.destructureDelimiter}if(options.delimited){workarray=destructuredSource.split(options.delimiter);workarray.pop()}else{workarray=destructuredSource}len=workarray.length;for(idx=0;idx=cdlen)doneWithPartition=true}if(pageLen>=this.options.pageSize)doneWithPage=true;if(!doneWithPage||doneWithPartition){pageBuilder+=this.options.delimiter;pageLen+=delimlen}if(doneWithPartition||doneWithPage){this.adapter.saveDatabase(keyname,pageBuilder,pageSaveCallback);return}}};function LokiFsAdapter(){this.fs=require("fs")}LokiFsAdapter.prototype.loadDatabase=function loadDatabase(dbname,callback){var self=this;this.fs.stat(dbname,function(err,stats){if(!err&&stats.isFile()){self.fs.readFile(dbname,{encoding:"utf8"},function readFileCallback(err,data){if(err){callback(new Error(err))}else{callback(data)}})}else{callback(null)}})};LokiFsAdapter.prototype.saveDatabase=function saveDatabase(dbname,dbstring,callback){var self=this;var tmpdbname=dbname+"~";this.fs.writeFile(tmpdbname,dbstring,function writeFileCallback(err){if(err){callback(new Error(err))}else{self.fs.rename(tmpdbname,dbname,callback)}})};LokiFsAdapter.prototype.deleteDatabase=function deleteDatabase(dbname,callback){this.fs.unlink(dbname,function deleteDatabaseCallback(err){if(err){callback(new Error(err))}else{callback()}})};function LokiLocalStorageAdapter(){}LokiLocalStorageAdapter.prototype.loadDatabase=function loadDatabase(dbname,callback){if(localStorageAvailable()){callback(localStorage.getItem(dbname))}else{callback(new Error("localStorage is not available"))}};LokiLocalStorageAdapter.prototype.saveDatabase=function saveDatabase(dbname,dbstring,callback){if(localStorageAvailable()){localStorage.setItem(dbname,dbstring);callback(null)}else{callback(new Error("localStorage is not available"))}};LokiLocalStorageAdapter.prototype.deleteDatabase=function deleteDatabase(dbname,callback){if(localStorageAvailable()){localStorage.removeItem(dbname);callback(null)}else{callback(new Error("localStorage is not available"))}};Loki.prototype.throttledSaveDrain=function(callback,options){var self=this;var now=(new Date).getTime();if(!this.throttledSaves){callback(true)} +options=options||{};if(!options.hasOwnProperty("recursiveWait")){options.recursiveWait=true}if(!options.hasOwnProperty("recursiveWaitLimit")){options.recursiveWaitLimit=false}if(!options.hasOwnProperty("recursiveWaitLimitDuration")){options.recursiveWaitLimitDuration=2e3}if(!options.hasOwnProperty("started")){options.started=(new Date).getTime()}if(this.throttledSaves&&this.throttledSavePending){if(options.recursiveWait){this.throttledCallbacks.push(function(){if(self.throttledSavePending){if(options.recursiveWaitLimit&&now-options.started>options.recursiveWaitLimitDuration){callback(false);return}self.throttledSaveDrain(callback,options);return}else{callback(true);return}})}else{this.throttledCallbacks.push(callback);return}}else{callback(true)}};Loki.prototype.loadDatabaseInternal=function(options,callback){var cFun=callback||function(err,data){if(err){throw err}},self=this;if(this.persistenceAdapter!==null){this.persistenceAdapter.loadDatabase(this.filename,function loadDatabaseCallback(dbString){if(typeof dbString==="string"){var parseSuccess=false;try{self.loadJSON(dbString,options||{});parseSuccess=true}catch(err){cFun(err)}if(parseSuccess){cFun(null);self.emit("loaded","database "+self.filename+" loaded")}}else{if(!dbString){cFun(null);self.emit("loaded","empty database "+self.filename+" loaded");return}if(dbString instanceof Error){cFun(dbString);return}if(typeof dbString==="object"){self.loadJSONObject(dbString,options||{});cFun(null);self.emit("loaded","database "+self.filename+" loaded");return}cFun("unexpected adapter response : "+dbString)}})}else{cFun(new Error("persistenceAdapter not configured"))}};Loki.prototype.loadDatabase=function(options,callback){var self=this;if(!this.throttledSaves){this.loadDatabaseInternal(options,callback);return}this.throttledSaveDrain(function(success){if(success){self.throttledSavePending=true;self.loadDatabaseInternal(options,function(err){if(self.throttledCallbacks.length===0){self.throttledSavePending=false}else{self.saveDatabase()}if(typeof callback==="function"){callback(err)}});return}else{if(typeof callback==="function"){callback(new Error("Unable to pause save throttling long enough to read database"))}}},options)};Loki.prototype.saveDatabaseInternal=function(callback){var cFun=callback||function(err){if(err){throw err}return},self=this;if(this.persistenceAdapter!==null){if(this.persistenceAdapter.mode==="reference"&&typeof this.persistenceAdapter.exportDatabase==="function"){this.persistenceAdapter.exportDatabase(this.filename,this.copy({removeNonSerializable:true}),function exportDatabaseCallback(err){self.autosaveClearFlags();cFun(err)})}else{self.autosaveClearFlags();this.persistenceAdapter.saveDatabase(this.filename,self.serialize(),function saveDatabasecallback(err){cFun(err)})}}else{cFun(new Error("persistenceAdapter not configured"))}};Loki.prototype.saveDatabase=function(callback){if(!this.throttledSaves){this.saveDatabaseInternal(callback);return}if(this.throttledSavePending){this.throttledCallbacks.push(callback);return}var localCallbacks=this.throttledCallbacks;this.throttledCallbacks=[];localCallbacks.unshift(callback);this.throttledSavePending=true;var self=this;this.saveDatabaseInternal(function(err){self.throttledSavePending=false;localCallbacks.forEach(function(pcb){if(typeof pcb==="function"){setTimeout(function(){pcb(err)},1)}});if(self.throttledCallbacks.length>0){self.saveDatabase()}})};Loki.prototype.save=Loki.prototype.saveDatabase;Loki.prototype.deleteDatabase=function(options,callback){var cFun=callback||function(err,data){if(err){throw err}};if(typeof options==="function"&&!callback){cFun=options}if(this.persistenceAdapter!==null){this.persistenceAdapter.deleteDatabase(this.filename,function deleteDatabaseCallback(err){cFun(err)})}else{cFun(new Error("persistenceAdapter not configured"))}};Loki.prototype.autosaveDirty=function(){for(var idx=0;idx0){this.filteredrows=[]}this.filterInitialized=false;return this};Resultset.prototype.toJSON=function(){var copy=this.copy();copy.collection=null;return copy};Resultset.prototype.limit=function(qty){if(!this.filterInitialized&&this.filteredrows.length===0){this.filteredrows=this.collection.prepareFullDocIndex()}var rscopy=new Resultset(this.collection);rscopy.filteredrows=this.filteredrows.slice(0,qty);rscopy.filterInitialized=true;return rscopy};Resultset.prototype.offset=function(pos){if(!this.filterInitialized&&this.filteredrows.length===0){this.filteredrows=this.collection.prepareFullDocIndex()}var rscopy=new Resultset(this.collection);rscopy.filteredrows=this.filteredrows.slice(pos);rscopy.filterInitialized=true;return rscopy};Resultset.prototype.copy=function(){var result=new Resultset(this.collection);if(this.filteredrows.length>0){result.filteredrows=this.filteredrows.slice()}result.filterInitialized=this.filterInitialized;return result};Resultset.prototype.branch=Resultset.prototype.copy;Resultset.prototype.transform=function(transform,parameters){var idx,step,rs=this;if(typeof transform==="string"){if(this.collection.transforms.hasOwnProperty(transform)){transform=this.collection.transforms[transform]}}if(typeof transform!=="object"||!Array.isArray(transform)){throw new Error("Invalid transform")}if(typeof parameters!=="undefined"){transform=Utils.resolveTransformParams(transform,parameters)}for(idx=0;idxobj2[propname])return 1;if(obj1[propname]1){return this.find({$and:filters},firstOnly)}}if(!property||queryObject==="getAll"){if(firstOnly){this.filteredrows=this.collection.data.length>0?[0]:[];this.filterInitialized=true}return this}if(property==="$and"||property==="$or"){this[property](queryObjectOp);if(firstOnly&&this.filteredrows.length>1){this.filteredrows=this.filteredrows.slice(0,1)}return this}if(queryObjectOp===null||(typeof queryObjectOp!=="object"||queryObjectOp instanceof Date)){operator="$eq";value=queryObjectOp}else if(typeof queryObjectOp==="object"){for(key in queryObjectOp){if(hasOwnProperty.call(queryObjectOp,key)){operator=key;value=queryObjectOp[key];break}}}else{throw new Error("Do not know what you want to do.")}if(operator==="$regex"){if(Array.isArray(value)){value=new RegExp(value[0],value[1])}else if(!(value instanceof RegExp)){value=new RegExp(value)}}var usingDotNotation=property.indexOf(".")!==-1;var doIndexCheck=!usingDotNotation&&!this.filterInitialized;if(doIndexCheck&&this.collection.binaryIndices[property]&&indexedOps[operator]){if(this.collection.adaptiveBinaryIndices!==true){this.collection.ensureIndex(property)}searchByIndex=true;index=this.collection.binaryIndices[property]}var fun=LokiOps[operator];var t=this.collection.data;var i=0,len=0;var filter,rowIdx=0;if(this.filterInitialized){filter=this.filteredrows;len=filter.length;if(usingDotNotation){property=property.split(".");for(i=0;i=0){this.filterPipeline[idx]=filter;return this.reapplyFilters()}this.cachedresultset=null;if(this.options.persistent){this.resultdata=[];this.resultsdirty=true}this._addFilter(filter);if(this.sortFunction||this.sortCriteria){this.queueSortPhase()}else{this.queueRebuildEvent()}return this};DynamicView.prototype.applyFind=function(query,uid){this.applyFilter({type:"find",val:query,uid:uid});return this};DynamicView.prototype.applyWhere=function(fun,uid){this.applyFilter({type:"where",val:fun,uid:uid});return this};DynamicView.prototype.removeFilter=function(uid){var idx=this._indexOfFilterWithId(uid);if(idx<0){throw new Error("Dynamic view does not contain a filter with ID: "+uid)}this.filterPipeline.splice(idx,1);this.reapplyFilters();return this};DynamicView.prototype.count=function(){if(this.resultsdirty){this.resultdata=this.resultset.data()}return this.resultset.count()};DynamicView.prototype.data=function(options){if(this.sortDirty||this.resultsdirty){this.performSortPhase({suppressRebuildEvent:true})}return this.options.persistent?this.resultdata:this.resultset.data(options)};DynamicView.prototype.queueRebuildEvent=function(){if(this.rebuildPending){return}this.rebuildPending=true;var self=this;setTimeout(function(){if(self.rebuildPending){self.rebuildPending=false;self.emit("rebuild",self)}},this.options.minRebuildInterval)};DynamicView.prototype.queueSortPhase=function(){if(this.sortDirty){return}this.sortDirty=true;var self=this;if(this.options.sortPriority==="active"){setTimeout(function(){self.performSortPhase()},this.options.minRebuildInterval)}else{this.queueRebuildEvent()}};DynamicView.prototype.performSortPhase=function(options){if(!this.sortDirty&&!this.resultsdirty){return}options=options||{};if(this.sortDirty){if(this.sortFunction){this.resultset.sort(this.sortFunction)}else if(this.sortCriteria){this.resultset.compoundsort(this.sortCriteria)}this.sortDirty=false}if(this.options.persistent){this.resultdata=this.resultset.data();this.resultsdirty=false}if(!options.suppressRebuildEvent){this.emit("rebuild",this)}};DynamicView.prototype.evaluateDocument=function(objIndex,isNew){if(!this.resultset.filterInitialized){if(this.options.persistent){this.resultdata=this.resultset.data()}if(this.sortFunction||this.sortCriteria){this.queueSortPhase()}else{this.queueRebuildEvent()}return}var ofr=this.resultset.filteredrows;var oldPos=isNew?-1:ofr.indexOf(+objIndex);var oldlen=ofr.length;var evalResultset=new Resultset(this.collection);evalResultset.filteredrows=[objIndex];evalResultset.filterInitialized=true;var filter;for(var idx=0,len=this.filterPipeline.length;idxobjIndex){ofr[idx]--}}};DynamicView.prototype.mapReduce=function(mapFunction,reduceFunction){try{return reduceFunction(this.data().map(mapFunction))}catch(err){throw err}};function Collection(name,options){this.name=name;this.data=[];this.idIndex=[];this.binaryIndices={};this.constraints={unique:{},exact:{}};this.uniqueNames=[];this.transforms={};this.objType=name;this.dirty=true;this.cachedIndex=null;this.cachedBinaryIndex=null;this.cachedData=null;var self=this;options=options||{};if(options.hasOwnProperty("unique")){if(!Array.isArray(options.unique)){options.unique=[options.unique]}options.unique.forEach(function(prop){self.uniqueNames.push(prop);self.constraints.unique[prop]=new UniqueIndex(prop)})}if(options.hasOwnProperty("exact")){options.exact.forEach(function(prop){self.constraints.exact[prop]=new ExactIndex(prop)})}this.adaptiveBinaryIndices=options.hasOwnProperty("adaptiveBinaryIndices")?options.adaptiveBinaryIndices:true;this.transactional=options.hasOwnProperty("transactional")?options.transactional:false;this.cloneObjects=options.hasOwnProperty("clone")?options.clone:false;this.cloneMethod=options.hasOwnProperty("cloneMethod")?options.cloneMethod:"parse-stringify";this.asyncListeners=options.hasOwnProperty("asyncListeners")?options.asyncListeners:false;this.disableMeta=options.hasOwnProperty("disableMeta")?options.disableMeta:false;this.disableChangesApi=options.hasOwnProperty("disableChangesApi")?options.disableChangesApi:true;this.disableDeltaChangesApi=options.hasOwnProperty("disableDeltaChangesApi")?options.disableDeltaChangesApi:true;if(this.disableChangesApi){this.disableDeltaChangesApi=true}this.autoupdate=options.hasOwnProperty("autoupdate")?options.autoupdate:false;this.serializableIndices=options.hasOwnProperty("serializableIndices")?options.serializableIndices:true;this.ttl={age:null,ttlInterval:null,daemon:null};this.setTTL(options.ttl||-1,options.ttlInterval);this.maxId=0;this.DynamicViews=[];this.events={insert:[],update:[],"pre-insert":[],"pre-update":[],close:[],flushbuffer:[],error:[],delete:[],warning:[]};this.changes=[];this.ensureId();var indices=[];if(options&&options.indices){if(Object.prototype.toString.call(options.indices)==="[object Array]"){indices=options.indices}else if(typeof options.indices==="string"){indices=[options.indices]}else{throw new TypeError("Indices needs to be a string or an array of strings")}}for(var idx=0;idx=0||propertyName=="$loki"||propertyName=="meta"){delta[propertyName]=newObject[propertyName]}else{var propertyDelta=getObjectDelta(oldObject[propertyName],newObject[propertyName]);if(typeof propertyDelta!=="undefined"&&propertyDelta!={}){delta[propertyName]=propertyDelta}}}}return Object.keys(delta).length===0?undefined:delta}else{return oldObject===newObject?undefined:newObject}}this.getObjectDelta=getObjectDelta;function flushChanges(){self.changes=[]}this.getChanges=function(){return self.changes};this.flushChanges=flushChanges;function insertMeta(obj){var len,idx;if(self.disableMeta||!obj){return}if(Array.isArray(obj)){len=obj.length;for(idx=0;idx1){options.randomSamplingFactor=.1}var valid=true,idx,iter,pos,len,biv;if(!this.binaryIndices.hasOwnProperty(property)){throw new Error("called checkIndex on property without an index: "+property)}if(!this.adaptiveBinaryIndices){this.ensureIndex(property)}biv=this.binaryIndices[property].values;len=biv.length;if(len!==this.data.length){if(options.repair){this.ensureIndex(property,true)}return false}if(len===0){return true}if(len===1){valid=biv[0]===0}if(options.randomSampling){if(!LokiOps.$lte(this.data[biv[0]][property],this.data[biv[1]][property])){valid=false}if(!LokiOps.$lte(this.data[biv[len-2]][property],this.data[biv[len-1]][property])){valid=false}if(valid){iter=Math.floor((len-1)*options.randomSamplingFactor);for(idx=0;idx0;if(adaptiveBatchOverride){this.adaptiveBinaryIndices=false}for(k;k>1;id=typeof id==="number"?id:parseInt(id,10);if(isNaN(id)){throw new TypeError("Passed id is not an integer")}while(data[min]>1;if(data[mid]dataPosition){index[idx]--}}};Collection.prototype.calculateRangeStart=function(prop,val,adaptive){var rcd=this.data;var index=this.binaryIndices[prop].values;var min=0;var max=index.length-1;var mid=0;if(index.length===0){return-1}var minVal=rcd[index[min]][prop];var maxVal=rcd[index[max]][prop];while(min>1;if(ltHelper(rcd[index[mid]][prop],val,false)){min=mid+1}else{max=mid}}var lbound=min;if(aeqHelper(val,rcd[index[lbound]][prop])){return lbound}if(ltHelper(val,rcd[index[lbound]][prop],false)){return adaptive?lbound:lbound-1}return adaptive?lbound+1:lbound};Collection.prototype.calculateRangeEnd=function(prop,val){var rcd=this.data;var index=this.binaryIndices[prop].values;var min=0;var max=index.length-1;var mid=0;if(index.length===0){return-1}var minVal=rcd[index[min]][prop];var maxVal=rcd[index[max]][prop];while(min>1;if(ltHelper(val,rcd[index[mid]][prop],false)){max=mid}else{min=mid+1}}var ubound=max;if(aeqHelper(val,rcd[index[ubound]][prop])){return ubound}if(gtHelper(val,rcd[index[ubound]][prop],false)){return ubound+1}if(aeqHelper(val,rcd[index[ubound-1]][prop])){return ubound-1}return ubound};Collection.prototype.calculateRange=function(op,prop,val){var rcd=this.data;var index=this.binaryIndices[prop].values;var min=0;var max=index.length-1;var mid=0;var lbound,lval;var ubound,uval;if(rcd.length===0){return[0,-1]}var minVal=rcd[index[min]][prop];var maxVal=rcd[index[max]][prop];switch(op){case"$eq":case"$aeq":if(ltHelper(val,minVal,false)||gtHelper(val,maxVal,false)){return[0,-1]}break;case"$dteq":if(ltHelper(val,minVal,false)||gtHelper(val,maxVal,false)){return[0,-1]}break;case"$gt":if(gtHelper(val,maxVal,true)){return[0,-1]}if(gtHelper(minVal,val,false)){return[min,max]}break;case"$gte":if(gtHelper(val,maxVal,false)){return[0,-1]}if(gtHelper(minVal,val,true)){return[min,max]}break;case"$lt":if(ltHelper(val,minVal,true)){return[0,-1]}if(ltHelper(maxVal,val,false)){return[min,max]}break;case"$lte":if(ltHelper(val,minVal,false)){return[0,-1]}if(ltHelper(maxVal,val,true)){return[min,max]}break;case"$between":if(gtHelper(val[0],maxVal,false)){return[0,-1]}if(ltHelper(val[1],minVal,false)){return[0,-1]}lbound=this.calculateRangeStart(prop,val[0]);ubound=this.calculateRangeEnd(prop,val[1]);if(lbound<0)lbound++;if(ubound>max)ubound--;if(!gtHelper(rcd[index[lbound]][prop],val[0],true))lbound++;if(!ltHelper(rcd[index[ubound]][prop],val[1],true))ubound--;if(ubounddeepProperty(this.data[i],field,deep)){min=deepProperty(this.data[i],field,deep);result.index=this.data[i].$loki}}else{min=deepProperty(this.data[i],field,deep);result.index=this.data[i].$loki}}result.value=min;return result};Collection.prototype.extractNumerical=function(field){return this.extract(field).map(parseBase10).filter(Number).filter(function(n){return!isNaN(n)})};Collection.prototype.avg=function(field){return average(this.extractNumerical(field))};Collection.prototype.stdDev=function(field){return standardDeviation(this.extractNumerical(field))};Collection.prototype.mode=function(field){var dict={},data=this.extract(field);data.forEach(function(obj){if(dict[obj]){dict[obj]+=1}else{dict[obj]=1}});var max,prop,mode;for(prop in dict){if(max){if(max0){root=root[pieces.shift()]}return root}function binarySearch(array,item,fun){var lo=0,hi=array.length,compared,mid;while(lo>1;compared=fun.apply(null,[item,array[mid]]);if(compared===0){return{found:true,index:mid}}else if(compared<0){hi=mid}else{lo=mid+1}}return{found:false,index:hi}}function BSonSort(fun){return function(array,item){return binarySearch(array,item,fun)}}function KeyValueStore(){}KeyValueStore.prototype={keys:[],values:[],sort:function(a,b){return ab?1:0},setSort:function(fun){this.bs=new BSonSort(fun)},bs:function(){return new BSonSort(this.sort)},set:function(key,value){var pos=this.bs(this.keys,key);if(pos.found){this.values[pos.index]=value}else{this.keys.splice(pos.index,0,key);this.values.splice(pos.index,0,value)}},get:function(key){return this.values[binarySearch(this.keys,key,this.sort).index]}};function UniqueIndex(uniqueField){this.field=uniqueField;this.keyMap={};this.lokiMap={}}UniqueIndex.prototype.keyMap={};UniqueIndex.prototype.lokiMap={};UniqueIndex.prototype.set=function(obj){var fieldValue=obj[this.field];if(fieldValue!==null&&typeof fieldValue!=="undefined"){if(this.keyMap[fieldValue]){throw new Error("Duplicate key for property "+this.field+": "+fieldValue)}else{this.keyMap[fieldValue]=obj;this.lokiMap[obj.$loki]=fieldValue}}};UniqueIndex.prototype.get=function(key){return this.keyMap[key]};UniqueIndex.prototype.byId=function(id){return this.keyMap[this.lokiMap[id]]};UniqueIndex.prototype.update=function(obj,doc){if(this.lokiMap[obj.$loki]!==doc[this.field]){var old=this.lokiMap[obj.$loki];this.set(doc);this.keyMap[old]=undefined}else{this.keyMap[obj[this.field]]=doc}};UniqueIndex.prototype.remove=function(key){var obj=this.keyMap[key];if(obj!==null&&typeof obj!=="undefined"){this.keyMap[key]=undefined;this.lokiMap[obj.$loki]=undefined}else{throw new Error("Key is not in unique index: "+this.field)}};UniqueIndex.prototype.clear=function(){this.keyMap={};this.lokiMap={}};function ExactIndex(exactField){this.index={};this.field=exactField}ExactIndex.prototype={set:function add(key,val){if(this.index[key]){this.index[key].push(val)}else{this.index[key]=[val]}},remove:function remove(key,val){var idxSet=this.index[key];for(var i in idxSet){if(idxSet[i]==val){idxSet.splice(i,1)}}if(idxSet.length<1){this.index[key]=undefined}},get:function get(key){return this.index[key]},clear:function clear(key){this.index={}}};function SortedIndex(sortedField){this.field=sortedField}SortedIndex.prototype={keys:[],values:[],sort:function(a,b){return ab?1:0},bs:function(){return new BSonSort(this.sort)},setSort:function(fun){this.bs=new BSonSort(fun)},set:function(key,value){var pos=binarySearch(this.keys,key,this.sort);if(pos.found){this.values[pos.index].push(value)}else{this.keys.splice(pos.index,0,key);this.values.splice(pos.index,0,[value])}},get:function(key){var bsr=binarySearch(this.keys,key,this.sort);if(bsr.found){return this.values[bsr.index]}else{return[]}},getLt:function(key){var bsr=binarySearch(this.keys,key,this.sort);var pos=bsr.index;if(bsr.found)pos--;return this.getAll(key,0,pos)},getGt:function(key){var bsr=binarySearch(this.keys,key,this.sort);var pos=bsr.index;if(bsr.found)pos++;return this.getAll(key,pos,this.keys.length)},getAll:function(key,start,end){var results=[];for(var i=start;i b; + }, + + $jgte: function (a, b) { + return a >= b; + }, + + $jlt: function (a, b) { + return a < b; + }, + + $jlte: function (a, b) { + return a <= b; + }, + // ex : coll.find({'orderCount': {$between: [10, 50]}}); $between: function (a, vals) { if (a === undefined || a === null) return false; @@ -2973,60 +2991,28 @@ * var results = users.chain().simplesort('age').data(); */ Resultset.prototype.simplesort = function (propname, options) { - var dc, frl, eff; + var eff, + dc = this.collection.data.length, + frl = this.filteredrows.length, + hasBinaryIndex = this.collection.binaryIndices.hasOwnProperty(propname); - if (typeof (options) === 'undefined') { + if (typeof (options) === 'undefined' || options === false) { options = { desc: false }; } if (options === true) { options = { desc: true }; } - if (options === false) { - options = { desc: false }; - } - - // If already filtered, but we want to leverage binary index on sort. - // This will use custom array intection algorithm. - if (!options.disableIndexIntersect && this.collection.binaryIndices.hasOwnProperty(propname) && this.filterInitialized) { - - dc = this.collection.data.length; - frl = this.filteredrows.length; - eff = dc/frl; - - // anything more than ratio of 10:1 (total documents/current results) should use old sort code path - // So we will only use array intersection if you have more than 10% of total docs in your current resultset. - if (eff <= 10 || options.forceIndexIntersect) { - var idx, len=this.filteredrows.length, fr=this.filteredrows; - var io = {}; - // set up hashobject for simple 'inclusion test' with existing (filtered) results - for(idx=0; idx obj2[propname]) return 1; - if (obj1[propname] < obj2[propname]) return -1; - }); - } - - // if this has no filters applied, just we need to populate filteredrows first - if (!this.filterInitialized && this.filteredrows.length === 0) { - // if we have a binary index and no other filters applied, we can use that instead of sorting (again) + + // otherwise no filters applied implies all documents, so we need to populate filteredrows first + + // if we have a binary index, we can just use that instead of sorting (again) if (this.collection.binaryIndices.hasOwnProperty(propname)) { // make sure index is up-to-date this.collection.ensureIndex(propname); @@ -3042,10 +3028,56 @@ } // otherwise initialize array for sort below else { + // build full document index (to be sorted subsequently) this.filteredrows = this.collection.prepareFullDocIndex(); } } + // otherwise we had results to begin with, see if we qualify for index intercept optimization + else { + + // If already filtered, but we want to leverage binary index on sort. + // This will use custom array intection algorithm. + if (!options.disableIndexIntersect && hasBinaryIndex) { + + // calculate filter efficiency + eff = dc/frl; + + // anything more than ratio of 10:1 (total documents/current results) should use old sort code path + // So we will only use array intersection if you have more than 10% of total docs in your current resultset. + if (eff <= 10 || options.forceIndexIntersect) { + var idx, fr=this.filteredrows; + var io = {}; + // set up hashobject for simple 'inclusion test' with existing (filtered) results + for(idx=0; idx obj2[propname]) return 1; + if (obj1[propname] < obj2[propname]) return -1; + }); + } + // otherwise use loki sort which will return same results if column is indexed or not var wrappedComparer = (function (prop, desc, data) { var val1, val2, arr; diff --git a/tutorials/Indexing and Query performance.md b/tutorials/Indexing and Query performance.md index 21ee6b13..31b78be8 100644 --- a/tutorials/Indexing and Query performance.md +++ b/tutorials/Indexing and Query performance.md @@ -1,6 +1,14 @@ Loki.js has always been a fast, in-memory database solution. In fact, recent benchmarks indicate that its primary get() operation is about _1.4 million operations_ per second fast on a mid-range Core i5 running under node.js. The get() operation utilizes an auto generated '$loki' id column with its own auto generated binary index. If you wish to supply your own unique key, you can use add a single unique index to the collection to be used along with the collection.by() method. This method is every bit as fast as using the built in $loki id. So out of the gate if you intend to do single object lookups you get this performance. -Example object lookup specifying your own unique index : +### Example lookup using autogenerated $loki column : +```javascript +var users = db.addCollection("users"); +var resultObj = users.insert({username:"Heimdallr"}); + +// now that our object has been inserted, it will have a $loki property added onto it +var heimdallr = users.get(resultObj.$loki); +``` +### Example object lookup specifying your own unique index : ```javascript var users = db.addCollection("users", { unique: ['username'] @@ -9,36 +17,62 @@ var users = db.addCollection("users", { // after inserting records you might retrieve your record using coll.by() var result = users.by("username", "Heimdallr"); ``` -Example lookup using autogenerated $loki column : -```javascript -var users = db.addCollection("users"); -var resultObj = users.insert({username:"Heimdallr"}); -// now that our object has been inserted, it will have a $loki property added onto it -var heimdallr = users.get(resultObj.$loki); -``` +### 'Find' filtering A more versatile way to query is to use collection.find() which accepts a mongo-style query object. If you do not index the column you are searching against, you can expect about 20k ops/sec under node.js (browser performance may vary but this serves as a good order of magnitude). For most purposes that is probably more performance than is needed, but you can now apply loki.js binary indexes on your object properties as well. Using the collection.ensureIndex(propertyName) method, you can create an index which can be used by various find() operations such as collection.find(). For our test benchmark, this increased performance to about _500k ops/sec_. -These binary indices can match multiple results or ranges and you might apply your index similar to in this example : +Binary indices are can be used with range ops returning multiple document results for the given property/range. If you have applied a binary index to a property, you can utilize that index by calling collection.find() with a query object containing that property. The find() ops which are able to utilize binary indices include $eq, $aeq, $lt, $lte, $gt, $gte, $between, $in. + +> By default, if you have a binary index applied on a property and you insert a document containing a javascript Date object as a value for that property, loki will replace it with a serializable-safe epoch time format date (integer). This is to prevent the index from becoming corrupt if we were to save it as a Date and load it as a string (they should be sorted differently). If you do not intend to save your database (using entirely in memory), you may pass a 'serializableIndices:false' collection option during collection creation and we will not alter your dates. + +### Binary index example : ```javascript var coll = db.addCollection('users', { indices: ['location'] }); // after inserting records you might use equality or range ops, -// such as this implicit $eq op : +// such as this implicit (strict) $eq op : var results = users.find({ location: 'Himinbjörg' }); ``` -'Where' filters (javascript filter functions) should be used sparingly if performance is of concern. It is unable to utilize indexes, so performance will be no better than an unindexed find, and depending on the complexity of your filter function even less so. Unindex queries and where filters always require a full array scan but they can be useful if thousands of ops/sec are sufficient or if used later in a query chain or dynamic view filter pipeline with less penalty. - +### Loki Sorting and Ranges +> Native javascript provides == (abstract) equality, === (strict) equality, < (abstract) less than, > (abstract) greater than, etc. Javascript deals with alot of mixed types, so you might want a numeric 4 to be (abstractly) equal to string '4'. If you wanted to test for 'less than' 4, it would by default convert to string so if you did not want strings you would have to test using 'typeof' or other type detection to manually filter out other types. +Loki would prefer to deal with pure clean data but has had to evolve to support various levels of 'dirty' data which are found in the wild. As such we have tried to adapt our concept of 'ranges' as it pertains to mixed types on properties so that they accommodate mixed types and provide similar find() results whether you are using a binary index or not. +- All values in loki are interpreted by loki as being 'less than', 'greater than', or 'equal' to any other value for the purposes of our find() op. This is different than javascript, so loki establishes a unified range ordering among two values. +- 4 is $aeq (abstractly equal to) '4', as is '3' $lt 4, and 4 is $gte '3', so mixing numbers and 'number-like' strings range in loki find as numbers. +- 'Number-like' strings are kept in different range (with the numbers) than 'non-number-like' strings, so 9999 will be $lt '111asdf" +- objects are all equal (unless you use dot notation to query an objects properties) +- Dates are sorted as numbers as their epoch time... those numbers are large so generally at the high end of number range +- $type op can be used to filter out results which do not match a specific javascript type +- $finite op can be used to filter out results which are either 'number-like' or 'non-number-like' + +### 'Where' filters +'Where' filters (javascript filter functions) should be used sparingly if performance is of concern. They are unable to utilize indexes, so performance will be no better than an unindexed find, and depending on the complexity of your filter function even less so. Unindex queries and where filters always require a full array scan but they can be useful if thousands of ops/sec are sufficient or if used later in a query chain or dynamic view filter pipeline with less penalty. + +### Indexing in Query chains The Resultset class introduced method chaining as an option for querying. You might use this method chaining to apply several find operations in succession or mix find(), where(), and sort() operations into a sequential chained pipe. For simplicity, an example of this might be (where users is a collection object) : ```javascript users.chain().find(queryObj).where(queryFunc).simplesort('name').data(); ``` -Examining this statement, if queryObj (a mongo-style query object) were { 'age': { '$gt': 30 } }, then that age column would be best to apply an index on, and that find() chain operation should come first in the chain. In chained operations, only the first chained operation can utilize the indexes for filtering. If it filtered out a sufficient number of records, the impact of the (where) query function will be less. The overhead of maintaining the filtered result set reduces performance by about 20% over collection.find, but they enable much more versatility. In our benchmarks this is still about _400k ops/sec_. +Examining this statement, if queryObj (a mongo-style query object) were { 'age': { '$gt': 30 } }, then that age column would be best to apply an index on, and that find() chain operation should come first in the chain. In chained operations, only the first chained filter can utilize the indexes for filtering. If it filtered out a sufficient number of records, the impact of the (where) query function will be less. The overhead of maintaining the filtered result set reduces performance by about 20% over collection.find, but they enable much more versatility. In our benchmarks this is still about _400k ops/sec_. + +### Indexes and sorting +When no filters are applied and a binary index exists on (for example) a 'name' property, the following can fully utilize the binary index : +```javascript +coll.chain().simplesort('name').data(); +``` +If filtering has occurred we will detect whether we can leverage the index within an 'index intersect' algorithm to speed up the sorting over a typical loki sort. This 'index intersect' algorithm will only be enabled if your resultset has more than 10% of the total documents within its resultset, otherwise a standard loki sort will be determined to be the faster method. The performance advantages of 'index intersect' are somewhat inversely proportional to your filter quality, so it leverages the binary index to help to reduce 'worst case' sorting penalties. + +Loki sorting is used not just for sorting but for building the binary indices, but if you do not need it's more unified sorting across mixed types, you might be able to shave additional milliseconds off your sort call by calling : +```javascript +coll.chain().simplesort('name', { useJavascriptSorting: true }).data(); +``` +Which (if a binary index exists on that 'name' property) we will use the index intersect algorithm unless resultset has 10% or less total document count, at which point we will fallback to javascript sorting on the property passed to simplesort. If you do not have a binary index applied on that property, we would always use javascript sorting if that option were passed. + +## Dynamic View pipelines Dynamic Views behave similarly to resultsets in that you want to utilize an index, your first filter must be applied using diff --git a/tutorials/Query Examples.md b/tutorials/Query Examples.md index c09723b6..245e617b 100644 --- a/tutorials/Query Examples.md +++ b/tutorials/Query Examples.md @@ -89,6 +89,30 @@ var results = coll.find({'age': {'$lt': 40}}); ```javascript var results = coll.find({'age': {'$lte': 40}}); ``` +> +> Note : the above $gt, $gte, $lt, and $lte ops use 'loki' sorting which provides a unified range actoss mixed types +> and which return the same results whether the property is indexed or not. This is needed for binary indexes and +> for guarantees of results equality between indexed and non-indexed comparisons. +> +> If you do not expect to utilize a binary index and you expect that simple javascript comparisons are acceptable, +> we provide the following ops which (due to their simplified comparisons) may provide more optimal execution speeds. +> +> **$jgt** - filter (using simplified javascript comparison) for docs with property greater than provided value +> ```javascript +> var results = coll.find({'age': {'$jgt': 40}}); +> ``` +> **$jgte** - filter (using simplified javascript comparison) for docs with property greater than or equal to provided value +> ```javascript +> var results = coll.find({'age': {'$jgte': 40}}); +> ``` +> **$jlt** - filter (using simplified javascript comparison) for docs with property less than provided value +> ```javascript +> var results = coll.find({'age': {'$jlt': 40}}); +> ``` +> **$jlte** - filter (using simplified javascript comparison) for docs with property less than or equal to provided value +> ```javascript +> var results = coll.find({'age': {'$jlte': 40}}); +> ``` **$between** - filter for documents(s) with property between provided vals ```javascript // match users with count value between 50 and 75