From 11cff1a074b48ae19b0ac05e96cf0493c2afe358 Mon Sep 17 00:00:00 2001 From: Pieter Soudan Date: Sat, 4 Apr 2015 11:51:23 +0200 Subject: [PATCH 1/3] support paging in cache. Store the range of data that is allready loaded for a query and merge subsequent data payloads in history --- lib/client.js | 77 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/lib/client.js b/lib/client.js index 6c11495..506248f 100644 --- a/lib/client.js +++ b/lib/client.js @@ -13,12 +13,12 @@ SearchSource = function SearchSource(source, fields, options) { this._currentQueryDep = new Tracker.Dependency(); this._currentVersion = 0; this._loadedVersion = 0; -} +}; SearchSource.prototype._loadData = function(query, options) { var self = this; var version = 0; - if(this._canUseHistory(query)) { + if(this._canUseHistory(query, options)) { this._updateStore(this.history[query].data); this.metaData.set(this.history[query].metadata); self._storeDep.changed(); @@ -43,11 +43,25 @@ SearchSource.prototype._loadData = function(query, options) { } if(self.options.keepHistory) { - self.history[query] = {data: data, loaded: new Date(), metadata: metadata}; + var range = { + start: options.skip || 0 + }; + range.end = range.start + (options.limit || data.length); + self._mergeInHistory(query, { + data: data, + range: range, + loaded: new Date(), + metadata: metadata + }); } if(version > self._loadedVersion) { - self._updateStore(data); + if(self.options.keepHistory){ + //new data was already merge in history + self._updateStore(self.history[query].data); + } else { + self._updateStore(data); + } self._loadedVersion = version; } @@ -60,16 +74,59 @@ SearchSource.prototype._loadData = function(query, options) { } }; -SearchSource.prototype._canUseHistory = function(query) { - var historyItem = this.history[query]; - if(this.options.keepHistory && historyItem) { - var diff = Date.now() - historyItem.loaded.getTime(); - return diff < this.options.keepHistory; +SearchSource.prototype._canUseHistory = function(query, options) { + if(this.options.keepHistory) { + var historyItem = this.history[query]; + if(historyItem){ + //are limiting options within current range? + if(options.skip || options.limit){ + if(!options.skip){ + options.skip = 0; + } + if(!options.limit){ + //TODO: default limit + options.limit = 99999; + } + if( options.skip < historyItem.range.start || + (options.skip + options.limit) > historyItem.range.end){ + return false; + } + } + var diff = Date.now() - historyItem.loaded.getTime(); + return diff < this.options.keepHistory; + } } return false; }; +SearchSource.prototype._mergeInHistory = function(query, newData) { + var current = this.history[query]; + if(!current){ + this.history[query] = newData; + return; + } + if(current.range.start > newData.range.start){ + current.range.start = newData.range.start; + } + if(current.range.end < newData.range.end){ + current.range.end = newData.range.end; + } + + //check what new data needs to be pushed to history data + var currentIds = _.reduce(current.data, function(memo, d){ + memo[d._id] = true; + return memo; + }, {}); + + _.each(newData.data, function(newDoc){ + if(!currentIds[newDoc._id]){ + current.data.push(newDoc); + } + }); + this.history[query] = current; +}; + SearchSource.prototype._updateStore = function(data) { var self = this; var storeIds = _.pluck(this.store.find().fetch(), "_id"); @@ -193,7 +250,7 @@ SearchSource.prototype.getMetadata = function() { SearchSource.prototype.getCurrentQuery = function() { this._currentQueryDep.depend(); return this.currentQuery; -} +}; SearchSource.prototype.getStatus = function() { return this.status.get(); From eed3e6cc54697ce9712378bea35734cc27128c34 Mon Sep 17 00:00:00 2001 From: Pieter Soudan Date: Tue, 7 Apr 2015 10:06:34 +0200 Subject: [PATCH 2/3] make sure only data is fetched from a range that is not in cache --- lib/client.js | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/lib/client.js b/lib/client.js index 506248f..004c1fa 100644 --- a/lib/client.js +++ b/lib/client.js @@ -18,16 +18,24 @@ SearchSource = function SearchSource(source, fields, options) { SearchSource.prototype._loadData = function(query, options) { var self = this; var version = 0; + if(this._canUseHistory(query, options)) { this._updateStore(this.history[query].data); this.metaData.set(this.history[query].metadata); self._storeDep.changed(); - } else { - this.status.set({loading: true}); - version = ++this._currentVersion; - this._fetch(this.source, query, options, handleData); + return; + } + + if(options){ + if(this.options.keepHistory) { + options = this._changeOptionsForWhatsInCache(query, options); + } } + this.status.set({loading: true}); + version = ++this._currentVersion; + this._fetch(this.source, query, options, handleData); + function handleData(err, payload) { if(err) { self.status.set({error: err}); @@ -127,9 +135,30 @@ SearchSource.prototype._mergeInHistory = function(query, newData) { this.history[query] = current; }; +SearchSource.prototype._changeOptionsForWhatsInCache = function(query, options) { + var historyItem = this.history[query]; + if(!historyItem){ + return options; + } + + if(options.skip || options.limit){ + if(!options.skip){ + options.skip = 0; + } + if( options.skip < historyItem.range.end ){ + if(options.limit){ + options.limit -= (historyItem.range.end - options.skip); + } + options.skip = historyItem.range.end; + } + } + + return options; +}; + SearchSource.prototype._updateStore = function(data) { var self = this; - var storeIds = _.pluck(this.store.find().fetch(), "_id"); + var storeIds = _.pluck(this.store.find({}, {fields: {_id: true}}).fetch(), "_id"); var currentIds = []; data.forEach(function(item) { currentIds.push(item._id); From 649eb03fa7e7e4c3e12cc9179e1a7c4653c32654 Mon Sep 17 00:00:00 2001 From: Pieter Soudan Date: Tue, 7 Apr 2015 10:20:15 +0200 Subject: [PATCH 3/3] add pagination section to readme --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 5fb30f4..8f60362 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,38 @@ Finally we can invoke search queries by invoking following API. PackageSearch.search("the text to search"); ``` +### Pagination + +You can provide options to limit your search query and implement pagination into your search: `skip` and `limit`. SearchSource will also sanitize your limiting options to only fetch data from you backend that is not already in history (that is, if you enabled local history caching by configuring a value for keepInHistory). + +```js +PackageSearch.search("the text to search", { + skip: 0, + limit: 12 +}); +``` + +For example, when you do the following subsequent searches with the same text string to seacrh for +```js +[ + {skip: 0, limit: 24}, + {skip: 0, limit: 36}, + {skip: 12, limit: 36}, + {skip: 24, limit: 24}, +]; +``` + +SearchSource will send the following requests on your backend (when keepInhistory is enabled): +```js +[ + {skip: 0, limit: 24}, + {skip: 24, limit: 12}, + {skip: 36, limit: 12} +]; +``` + +To see an elaborate example on how to implement a paged search application, go to (http://infinitesearch.meteor.com). + ### Status You can get the status of the search source by invoking following API. It's reactive too.