added gzipResponse option to enable/disable storing of responses as g…

README.md

@@ -9,7 +9,7 @@ This tool was made to work with the popular [request](https://github.com/request
 
 First, you instantiate a **cachedRequest** instance by passing a **request** function, which is going to act as the requester for the uncached requests - you still need to `$npm install request` independently. - Then, you can use **cachedRequest** to perform your HTTP requests.
 
-The caching takes place in the filesystem, storing the responses as compressed gzipped files.
+The caching takes place in the filesystem, storing the responses as compressed gzipped files by default.
 
 Please note this will cache *everything*, so don't use it for making things like POST or PUT requests that you don't want to be cached.
 
@@ -67,6 +67,16 @@ When making a request, you must pass an `options` object as you can observe in t
     cachedRequest({url: 'https://www.google.com'}, callback); // should benefit from the cache if previously cached
     ```
 
+- `gzipResponse`: Flag to store responses as gzipped files on the filesystem. Default = true. Adds a `_gzipResponse` flag to the json headers file. 
+
+    ```javascript
+      var options = {
+        url: "https://www.google.com",
+        gzipResponse: false // downloaded files will not be gzipped when saved
+      };
+      cachedRequest(options, callback);
+    ```
+
 ##Can I use everything that comes with **request**?
 No, there's some things you can't use. For example, the shortcut functions `.get`, `.post`, `.put`, etc. are not available in **cached-request**. If you'd like to have them, this is a great opportunity to contribute!
 

lib/cached-request.js

@@ -10,7 +10,7 @@ var fs = require("graceful-fs")
 ,   zlib = require("zlib")
 ,   Transform = require("stream").Transform
 ,   EventEmitter = require("events").EventEmitter
-,   mkdirp = require('mkdirp');
+,   lo = require('lodash');
 
 util.inherits(Response, Transform);
 
@@ -33,6 +33,7 @@ function CachedRequest (request) {
   this.request = request;
   this.cacheDirectory = "/tmp/";
   this.ttl = 0;
+  this.gzipResponse = true;
 
   function _request () {
     return self.cachedRequest.apply(self, arguments);
@@ -70,8 +71,6 @@ CachedRequest.prototype.setCacheDirectory = function (cacheDirectory) {
   if (this.cacheDirectory.lastIndexOf("/") < this.cacheDirectory.length - 1) {
     this.cacheDirectory += "/";
   };
-  // Create directory path if it doesn't exist
-  mkdirp(this.cacheDirectory);
 };
 
 CachedRequest.prototype.handleError = function (error) {
@@ -114,6 +113,7 @@ CachedRequest.prototype.cachedRequest = function () {
   ,   args = arguments
   ,   options = args[0]
   ,   ttl = options.ttl || this.ttl
+  ,   gzipResponse = typeof(options.gzipResponse) === 'undefined' ? this.gzipResponse : options.gzipResponse
   ,   mustParseJSON = false
   ,   callback
   ,   headersReader
@@ -192,8 +192,10 @@ CachedRequest.prototype.cachedRequest = function () {
           //Emit the "response" event to the client sending the fake response
           requestMiddleware.emit("response", response);
 
+          var gzipResponse = response.headers._gzipResponse;
+
           var stream;
-          if (response.headers['content-encoding'] === 'gzip') {
+          if (response.headers['content-encoding'] === 'gzip' || ! gzipResponse) {
             stream = responseReader;
           } else {
             // Gunzip the response file
@@ -256,7 +258,12 @@ CachedRequest.prototype.cachedRequest = function () {
         response.on('error', function (error) {
           self.handleError(error);
         });
-        fs.writeFile(self.cacheDirectory + key + ".json", JSON.stringify(response.headers), function (error) {
+
+        // save metadata: response headers and gzipped flag
+        var meta = lo.clone(response.headers);
+        meta._gzipResponse = gzipResponse;
+
+        fs.writeFile(self.cacheDirectory + key + ".json", JSON.stringify(meta), function (error) {
           if (error) self.handleError(error);
         });
 
@@ -269,7 +276,7 @@ CachedRequest.prototype.cachedRequest = function () {
         contentEncoding = response.headers['content-encoding'] || '';
         contentEncoding = contentEncoding.trim().toLowerCase();
 
-        if (contentEncoding === 'gzip') {
+        if (contentEncoding === 'gzip' || ! gzipResponse) {
           response.on('error', function (error) {
             responseWriter.end();
           });

package.json

@@ -21,10 +21,12 @@
   ],
   "dependencies": {
     "graceful-fs": "^4.0.0",
+    "lodash": "^4.15.0",
     "mkdirp": "^0.5.1"
   },
   "devDependencies": {
     "chai": "^1.10.0",
+    "mmmagic": "^0.4.5",
     "mocha": "^2.1.0",
     "nock": "^0.52.4",
     "request": "^2.51.0",

test/test.js

@@ -4,7 +4,11 @@ var CachedRequest = require("../")
 ,   temp = require('temp').track()
 ,   Readable = require("stream").Readable
 ,   util = require("util")
-,   zlib = require("zlib");
+,   zlib = require("zlib")
+,   mmm = require('mmmagic')
+,   Magic = mmm.Magic
+,   path = require('path')
+,   fs = require('fs');
 
 util.inherits(MockedResponseStream, Readable);
 
@@ -171,6 +175,132 @@ describe("CachedRequest", function () {
         });
       });
     });
+
+    it("stores response un-gzip'd when gzipResponse option is disabled", function (done) {
+      var self = this;
+      var responseBody = {"a": 1, "b": {"c": 2}};
+      var options = {
+        uri: "http://ping.com/",
+        method: "POST",
+        json: {
+          a: 1
+        },
+        ttl: 5000,
+        gzipResponse: false
+      };
+
+      mock(options.method, 1, function () {
+        return new MockedResponseStream({}, JSON.stringify(responseBody));
+      });
+
+      this.cachedRequest(options, function (error, response, body) {
+        if (error) return done(error);
+        expect(response.statusCode).to.equal(200);
+        expect(response.headers["x-from-cache"]).to.not.exist;
+        expect(body).to.deep.equal(responseBody);
+        var magic = new Magic(mmm.MAGIC_MIME_TYPE)
+        ,   cacheDir = self.cachedRequest.getValue('cacheDirectory')
+        ,   basename = self.cachedRequest.getValue('hashKey')(JSON.stringify(self.cachedRequest.getValue('normalizeOptions')(options)))
+        ,   filepath = cacheDir + basename
+        ,   metaFilepath = filepath + '.json';
+
+        // wait for JSON file to written + flushed
+        setTimeout(function(){
+            var meta = JSON.parse(fs.readFileSync(metaFilepath));
+
+            expect(meta._gzipResponse).to.equal(false);
+
+            magic.detectFile(filepath, function(err, result) {
+                if (err) throw err;
+                expect(result).to.equal('text/plain');
+                done();
+            });
+         }, 25);
+      });
+    });
+
+    it("stores response gzip'd when gzipResponse option is omitted", function (done) {
+      var self = this;
+      var responseBody = {"a": 1, "b": {"c": 2}};
+      var options = {
+        uri: "http://ping.com/",
+        method: "POST",
+        json: {
+          a: 1
+        },
+        ttl: 5000
+      };
+
+      mock(options.method, 1, function () {
+        return new MockedResponseStream({}, JSON.stringify(responseBody));
+      });
+
+      this.cachedRequest(options, function (error, response, body) {
+        if (error) return done(error);
+        expect(response.statusCode).to.equal(200);
+        expect(response.headers["x-from-cache"]).to.not.exist;
+        expect(body).to.deep.equal(responseBody);
+        var magic = new Magic(mmm.MAGIC_MIME_TYPE)
+        ,   cacheDir = self.cachedRequest.getValue('cacheDirectory')
+        ,   basename = self.cachedRequest.getValue('hashKey')(JSON.stringify(self.cachedRequest.getValue('normalizeOptions')(options)))
+        ,   filepath = cacheDir + basename
+        ,   metaFilepath = filepath + '.json';
+
+        // wait for JSON file to written + flushed
+        setTimeout(function(){ 
+            var meta = JSON.parse(fs.readFileSync(metaFilepath));
+
+            expect(meta._gzipResponse).to.equal(true);
+
+            magic.detectFile(filepath, function(err, result) {
+                if (err) throw err;
+                expect(result).to.equal('application/x-gzip'); 
+                done();
+            });
+         }, 25);
+      });
+    });
+
+    it("responds the same from the cache if gzipResponse option is enabled", function (done) {
+      var self = this;
+      var responseBody = 'foo';
+      var options = {
+        url: "http://ping.com/",
+        ttl: 5000,
+        encoding: null, // avoids messing with gzip responses so we can handle them
+        gzipResponse: true 
+      };
+
+      //Return gzip compressed response with valid content encoding header
+      mock("GET", 1, function () {
+        return new MockedResponseStream({}, responseBody).pipe(zlib.createGzip());
+      },
+      {
+        "Content-Encoding": "gzip"
+      });
+
+      this.cachedRequest(options, function (error, response, body) {
+        if (error) return done(error);
+        expect(response.statusCode).to.equal(200);
+        expect(response.headers["x-from-cache"]).to.not.exist;
+
+        zlib.gunzip(body, function (error, buffer) {
+          if (error) return done(error);
+          expect(buffer.toString()).to.deep.equal(responseBody);
+
+          self.cachedRequest(options, function (error, response, body) {
+            if (error) return done(error);
+            expect(response.statusCode).to.equal(200);
+            expect(response.headers["x-from-cache"]).to.equal(1);
+            zlib.gunzip(body, function (error, buffer) {
+              if (error) done(error);
+              expect(buffer.toString()).to.deep.equal(responseBody);
+              done();
+            });
+          });
+        });
+      });
+    });
   });
 
   describe("streaming", function () {
@@ -213,6 +343,45 @@ describe("CachedRequest", function () {
       });
     });
 
+    it("allows to use request as a stream when gzipResponse option is disabled", function (done) {
+      var self = this;
+      var responseBody = "";
+
+      for (var i = 0; i < 1000; i++) {
+        responseBody += "this is a long response body";
+      };
+
+      mock("GET", 1, function () {
+        return new MockedResponseStream({}, responseBody);
+      });
+
+      var options = {url: "http://ping.com/", ttl: 5000, gzipResponse: false};
+      var body = "";
+
+      //Make fresh request
+      this.cachedRequest(options)
+      .on("data", function (data) {
+          body += data;
+      })
+      .on("end", function () {
+        expect(body).to.equal(responseBody);
+        body = "";
+        //Make cached request
+        self.cachedRequest(options)
+        .on("response", function (response) {
+          expect(response.statusCode).to.equal(200);
+          expect(response.headers["x-from-cache"]).to.equal(1);
+          response.on("data", function (data) {
+            body += data;
+          })
+          .on("end", function () {
+            expect(body).to.equal(responseBody);
+            done();
+          });
+        });
+      });
+    });
+
     it("allows to use request with get extension method as a stream", function (done) {
       var self = this;
       var responseBody = "";
@@ -309,4 +478,4 @@ describe("CachedRequest", function () {
   after(function () {
     temp.cleanupSync();
   });
-});
+});