From 836cc28d3b109824748fd8a9f778df34b3ce91ec Mon Sep 17 00:00:00 2001 From: Timbo White <> Date: Wed, 7 Sep 2016 01:59:29 +0000 Subject: [PATCH] added gzipResponse option to enable/disable storing of responses as gzipped files --- README.md | 12 ++- lib/cached-request.js | 19 +++-- package.json | 2 + test/test.js | 173 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 197 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d3aa1cd..94ffe71 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This tool was made to work with the popular [request](https://github.com/request First, you instantiate a **cachedRequest** instance by passing a **request** function, which is going to act as the requester for the uncached requests - you still need to `$npm install request` independently. - Then, you can use **cachedRequest** to perform your HTTP requests. -The caching takes place in the filesystem, storing the responses as compressed gzipped files. +The caching takes place in the filesystem, storing the responses as compressed gzipped files by default. Please note this will cache *everything*, so don't use it for making things like POST or PUT requests that you don't want to be cached. @@ -67,6 +67,16 @@ When making a request, you must pass an `options` object as you can observe in t cachedRequest({url: 'https://www.google.com'}, callback); // should benefit from the cache if previously cached ``` +- `gzipResponse`: Flag to store responses as gzipped files on the filesystem. Default = true. Adds a `_gzipResponse` flag to the json headers file. + + ```javascript + var options = { + url: "https://www.google.com", + gzipResponse: false // downloaded files will not be gzipped when saved + }; + cachedRequest(options, callback); + ``` + ##Can I use everything that comes with **request**? No, there's some things you can't use. For example, the shortcut functions `.get`, `.post`, `.put`, etc. are not available in **cached-request**. If you'd like to have them, this is a great opportunity to contribute! diff --git a/lib/cached-request.js b/lib/cached-request.js index a5bd438..5ee4c72 100644 --- a/lib/cached-request.js +++ b/lib/cached-request.js @@ -10,7 +10,7 @@ var fs = require("graceful-fs") , zlib = require("zlib") , Transform = require("stream").Transform , EventEmitter = require("events").EventEmitter -, mkdirp = require('mkdirp'); +, lo = require('lodash'); util.inherits(Response, Transform); @@ -33,6 +33,7 @@ function CachedRequest (request) { this.request = request; this.cacheDirectory = "/tmp/"; this.ttl = 0; + this.gzipResponse = true; function _request () { return self.cachedRequest.apply(self, arguments); @@ -70,8 +71,6 @@ CachedRequest.prototype.setCacheDirectory = function (cacheDirectory) { if (this.cacheDirectory.lastIndexOf("/") < this.cacheDirectory.length - 1) { this.cacheDirectory += "/"; }; - // Create directory path if it doesn't exist - mkdirp(this.cacheDirectory); }; CachedRequest.prototype.handleError = function (error) { @@ -114,6 +113,7 @@ CachedRequest.prototype.cachedRequest = function () { , args = arguments , options = args[0] , ttl = options.ttl || this.ttl + , gzipResponse = typeof(options.gzipResponse) === 'undefined' ? this.gzipResponse : options.gzipResponse , mustParseJSON = false , callback , headersReader @@ -192,8 +192,10 @@ CachedRequest.prototype.cachedRequest = function () { //Emit the "response" event to the client sending the fake response requestMiddleware.emit("response", response); + var gzipResponse = response.headers._gzipResponse; + var stream; - if (response.headers['content-encoding'] === 'gzip') { + if (response.headers['content-encoding'] === 'gzip' || ! gzipResponse) { stream = responseReader; } else { // Gunzip the response file @@ -256,7 +258,12 @@ CachedRequest.prototype.cachedRequest = function () { response.on('error', function (error) { self.handleError(error); }); - fs.writeFile(self.cacheDirectory + key + ".json", JSON.stringify(response.headers), function (error) { + + // save metadata: response headers and gzipped flag + var meta = lo.clone(response.headers); + meta._gzipResponse = gzipResponse; + + fs.writeFile(self.cacheDirectory + key + ".json", JSON.stringify(meta), function (error) { if (error) self.handleError(error); }); @@ -269,7 +276,7 @@ CachedRequest.prototype.cachedRequest = function () { contentEncoding = response.headers['content-encoding'] || ''; contentEncoding = contentEncoding.trim().toLowerCase(); - if (contentEncoding === 'gzip') { + if (contentEncoding === 'gzip' || ! gzipResponse) { response.on('error', function (error) { responseWriter.end(); }); diff --git a/package.json b/package.json index ddcc7df..03db4ea 100644 --- a/package.json +++ b/package.json @@ -21,10 +21,12 @@ ], "dependencies": { "graceful-fs": "^4.0.0", + "lodash": "^4.15.0", "mkdirp": "^0.5.1" }, "devDependencies": { "chai": "^1.10.0", + "mmmagic": "^0.4.5", "mocha": "^2.1.0", "nock": "^0.52.4", "request": "^2.51.0", diff --git a/test/test.js b/test/test.js index 0edbd58..e53bee2 100644 --- a/test/test.js +++ b/test/test.js @@ -4,7 +4,11 @@ var CachedRequest = require("../") , temp = require('temp').track() , Readable = require("stream").Readable , util = require("util") -, zlib = require("zlib"); +, zlib = require("zlib") +, mmm = require('mmmagic') +, Magic = mmm.Magic +, path = require('path') +, fs = require('fs'); util.inherits(MockedResponseStream, Readable); @@ -171,6 +175,132 @@ describe("CachedRequest", function () { }); }); }); + + it("stores response un-gzip'd when gzipResponse option is disabled", function (done) { + var self = this; + var responseBody = {"a": 1, "b": {"c": 2}}; + var options = { + uri: "http://ping.com/", + method: "POST", + json: { + a: 1 + }, + ttl: 5000, + gzipResponse: false + }; + + mock(options.method, 1, function () { + return new MockedResponseStream({}, JSON.stringify(responseBody)); + }); + + this.cachedRequest(options, function (error, response, body) { + if (error) return done(error); + expect(response.statusCode).to.equal(200); + expect(response.headers["x-from-cache"]).to.not.exist; + expect(body).to.deep.equal(responseBody); + var magic = new Magic(mmm.MAGIC_MIME_TYPE) + , cacheDir = self.cachedRequest.getValue('cacheDirectory') + , basename = self.cachedRequest.getValue('hashKey')(JSON.stringify(self.cachedRequest.getValue('normalizeOptions')(options))) + , filepath = cacheDir + basename + , metaFilepath = filepath + '.json'; + + // wait for JSON file to written + flushed + setTimeout(function(){ + var meta = JSON.parse(fs.readFileSync(metaFilepath)); + + expect(meta._gzipResponse).to.equal(false); + + magic.detectFile(filepath, function(err, result) { + if (err) throw err; + expect(result).to.equal('text/plain'); + done(); + }); + }, 25); + }); + }); + + it("stores response gzip'd when gzipResponse option is omitted", function (done) { + var self = this; + var responseBody = {"a": 1, "b": {"c": 2}}; + var options = { + uri: "http://ping.com/", + method: "POST", + json: { + a: 1 + }, + ttl: 5000 + }; + + mock(options.method, 1, function () { + return new MockedResponseStream({}, JSON.stringify(responseBody)); + }); + + this.cachedRequest(options, function (error, response, body) { + if (error) return done(error); + expect(response.statusCode).to.equal(200); + expect(response.headers["x-from-cache"]).to.not.exist; + expect(body).to.deep.equal(responseBody); + var magic = new Magic(mmm.MAGIC_MIME_TYPE) + , cacheDir = self.cachedRequest.getValue('cacheDirectory') + , basename = self.cachedRequest.getValue('hashKey')(JSON.stringify(self.cachedRequest.getValue('normalizeOptions')(options))) + , filepath = cacheDir + basename + , metaFilepath = filepath + '.json'; + + // wait for JSON file to written + flushed + setTimeout(function(){ + var meta = JSON.parse(fs.readFileSync(metaFilepath)); + + expect(meta._gzipResponse).to.equal(true); + + magic.detectFile(filepath, function(err, result) { + if (err) throw err; + expect(result).to.equal('application/x-gzip'); + done(); + }); + }, 25); + }); + }); + + it("responds the same from the cache if gzipResponse option is enabled", function (done) { + var self = this; + var responseBody = 'foo'; + var options = { + url: "http://ping.com/", + ttl: 5000, + encoding: null, // avoids messing with gzip responses so we can handle them + gzipResponse: true + }; + + //Return gzip compressed response with valid content encoding header + mock("GET", 1, function () { + return new MockedResponseStream({}, responseBody).pipe(zlib.createGzip()); + }, + { + "Content-Encoding": "gzip" + }); + + this.cachedRequest(options, function (error, response, body) { + if (error) return done(error); + expect(response.statusCode).to.equal(200); + expect(response.headers["x-from-cache"]).to.not.exist; + + zlib.gunzip(body, function (error, buffer) { + if (error) return done(error); + expect(buffer.toString()).to.deep.equal(responseBody); + + self.cachedRequest(options, function (error, response, body) { + if (error) return done(error); + expect(response.statusCode).to.equal(200); + expect(response.headers["x-from-cache"]).to.equal(1); + zlib.gunzip(body, function (error, buffer) { + if (error) done(error); + expect(buffer.toString()).to.deep.equal(responseBody); + done(); + }); + }); + }); + }); + }); }); describe("streaming", function () { @@ -213,6 +343,45 @@ describe("CachedRequest", function () { }); }); + it("allows to use request as a stream when gzipResponse option is disabled", function (done) { + var self = this; + var responseBody = ""; + + for (var i = 0; i < 1000; i++) { + responseBody += "this is a long response body"; + }; + + mock("GET", 1, function () { + return new MockedResponseStream({}, responseBody); + }); + + var options = {url: "http://ping.com/", ttl: 5000, gzipResponse: false}; + var body = ""; + + //Make fresh request + this.cachedRequest(options) + .on("data", function (data) { + body += data; + }) + .on("end", function () { + expect(body).to.equal(responseBody); + body = ""; + //Make cached request + self.cachedRequest(options) + .on("response", function (response) { + expect(response.statusCode).to.equal(200); + expect(response.headers["x-from-cache"]).to.equal(1); + response.on("data", function (data) { + body += data; + }) + .on("end", function () { + expect(body).to.equal(responseBody); + done(); + }); + }); + }); + }); + it("allows to use request with get extension method as a stream", function (done) { var self = this; var responseBody = ""; @@ -309,4 +478,4 @@ describe("CachedRequest", function () { after(function () { temp.cleanupSync(); }); -}); \ No newline at end of file +});