Use Redis SCAN and MGET commands in ids() and all() (refs tj#203)

naholyr · Apr 25, 2017 · c20be96 · c20be96
1 parent da6cf26
commit c20be96
Show file tree

Hide file tree

Showing 2 changed files with 58 additions and 27 deletions.
diff --git a/Readme.md b/Readme.md
@@ -24,10 +24,10 @@ app.use(session({
 Options
 -------
 
-A Redis client is required. An existing client can be passed directly using the `client` param or created for you using the `host`, `port`, or `socket` params. 
+A Redis client is required. An existing client can be passed directly using the `client` param or created for you using the `host`, `port`, or `socket` params.
 
 - `client` An existing client
-- `host` Redis server hostname 
+- `host` Redis server hostname
 - `port` Redis server portno
 - `socket` Redis server unix_socket
 - `url` Redis server url
@@ -45,6 +45,7 @@ The following additional params may be included:
 	-	If `true`, a default logging function (`console.error`) is provided.
 	-	If a function, it is called anytime an error occurs (useful for custom logging)
 	-	If `false`, no logging occurs.
+-	`scanCount` Value used for *count* parameter in [Redis `SCAN` command](https://redis.io/commands/scan#the-count-option) (used in `ids()` and `all()` methods, defaults to 100).
 
 Any options not included in this list will be passed to the redis `createClient()` method directly.
 

diff --git a/lib/connect-redis.js b/lib/connect-redis.js
@@ -60,6 +60,9 @@ module.exports = function (session) {
 
     delete options.prefix;
 
+    this.scanCount = Number(options.scanCount) || 100;
+    delete options.scanCount;
+
     this.serializer = options.serializer || JSON;
 
     if (options.url) {
@@ -83,11 +86,11 @@ module.exports = function (session) {
       // which may need to write to a distributed log
       if(typeof options.logErrors != 'function'){
         options.logErrors = function (err) {
-  			  console.error('Warning: connect-redis reported a client error: ' + err);
-  		  };
+          console.error('Warning: connect-redis reported a client error: ' + err);
+        };
       }
-  		this.client.on('error', options.logErrors);
-  	}
+      this.client.on('error', options.logErrors);
+    }
 
     if (options.pass) {
       this.client.auth(options.pass, function (err) {
@@ -246,6 +249,43 @@ module.exports = function (session) {
     });
   };
 
+  /**
+   * Fetch all sessions' Redis keys using non-blocking SCAN command
+   *
+   * @param {Function} fn
+   * @api private
+   */
+
+  function allKeys (cb) {
+    var store = this;
+    var keysObj = {}; // Use an object to dedupe as scan can return duplicates
+    var pattern = store.prefix + '*';
+    var scanCount = store.scanCount;
+    debug('SCAN "%s"', pattern);
+    (function nextBatch (cursorId) {
+      store.client.scan(cursorId, 'match', pattern, 'count', scanCount, function (err, result) {
+        if (err) return cb(err);
+
+        var nextCursorId = result[0];
+        var keys = result[1];
+
+        debug('SCAN complete (next cursor = "%s")', nextCursorId);
+
+        keys.forEach(function (key) {
+          keysObj[key] = 1;
+        });
+
+        if (nextCursorId != 0) {
+          // next batch
+          return nextBatch(nextCursorId);
+        }
+
+        // end of cursor
+        return cb(null, Object.keys(keysObj));
+      });
+    })(0);
+  }
+
   /**
    * Fetch all sessions' ids
    *
@@ -255,14 +295,12 @@ module.exports = function (session) {
 
   RedisStore.prototype.ids = function (fn) {
     var store = this;
-    var pattern = store.prefix + '*';
     var prefixLength = store.prefix.length;
     if (!fn) fn = noop;
 
-    debug('KEYS "%s"', pattern);
-    store.client.keys(pattern, function (er, keys) {
-      if (er) return fn(er);
-      debug('KEYS complete');
+    allKeys(function (err, keys) {
+      if (err) return fn(err);
+
       keys = keys.map(function (key) {
         return key.substr(prefixLength);
       });
@@ -280,23 +318,14 @@ module.exports = function (session) {
 
   RedisStore.prototype.all = function (fn) {
     var store = this;
-    var pattern = store.prefix + '*';
     var prefixLength = store.prefix.length;
     if (!fn) fn = noop;
 
-    debug('KEYS "%s"', pattern);
-    store.client.keys(pattern, function (er, keys) {
-      if (er) return fn(er);
-      debug('KEYS complete');
-
-      var multi = store.client.multi();
+    allKeys(function (err, keys) {
+      if (err) return fn(err);
 
-      keys.forEach(function (key) {
-        multi.get(key);
-      });
-
-      multi.exec(function (er, sessions) {
-        if (er) return fn(er);
+      store.client.mget(keys, function (err, sessions) {
+        if (err) return fn(err);
 
         var result;
         try {
@@ -306,10 +335,11 @@ module.exports = function (session) {
             data.id = keys[index].substr(prefixLength);
             return data;
           });
-        } catch (er) {
-          return fn(er);
+        } catch (e) {
+          err = e;
         }
-        return fn(null, result);
+
+        return fn(err, result);
       });
     });
   };