diff --git a/README.md b/README.md index 767687bd0..c4c0afd73 100644 --- a/README.md +++ b/README.md @@ -790,7 +790,8 @@ request({url: 'https://www.google.com', verbose: true}, function (error, respons }, "response": { "statusCode": 200, - "httpVersion": "1.1" + "httpVersion": "1.1". + "downloadedBytes": 1234, }, "timingStart": 1552908287924, "timingStartTimer": 805.690674, diff --git a/lib/helpers.js b/lib/helpers.js index 5df296ace..835de6495 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -3,6 +3,7 @@ var jsonSafeStringify = require('json-stringify-safe') var crypto = require('crypto') var Buffer = require('safe-buffer').Buffer +var { Transform } = require('stream') var defer = typeof setImmediate === 'undefined' ? process.nextTick @@ -58,6 +59,23 @@ function version () { } } +class SizeTrackerStream extends Transform { + constructor (options) { + super(options) + this.size = 0 + } + + _transform (chunk, encoding, callback) { + this.size += chunk.length + this.push(chunk) + callback() + } + + _flush (callback) { + callback() + } +} + exports.safeStringify = safeStringify exports.md5 = md5 exports.isReadStream = isReadStream @@ -65,3 +83,4 @@ exports.toBase64 = toBase64 exports.copy = copy exports.version = version exports.defer = defer +exports.SizeTrackerStream = SizeTrackerStream diff --git a/lib/http2/request.js b/lib/http2/request.js index 9b5a33b92..ac6008463 100644 --- a/lib/http2/request.js +++ b/lib/http2/request.js @@ -135,6 +135,7 @@ class Http2Request extends EventEmitter { } this.stream.setEncoding(encoding) + return this } write (chunk) { @@ -142,7 +143,7 @@ class Http2Request extends EventEmitter { this._flushHeaders() } - this.stream.write(chunk) + return this.stream.write(chunk) } _flushHeaders () { @@ -167,6 +168,8 @@ class Http2Request extends EventEmitter { this._flushHeaders() } this.stream.pipe(dest) + + return dest } on (eventName, listener) { @@ -183,6 +186,8 @@ class Http2Request extends EventEmitter { this._flushHeaders() } this.stream.destroy() + + return this } end () { @@ -190,6 +195,8 @@ class Http2Request extends EventEmitter { this._flushHeaders() } this.stream.end() + + return this } setTimeout (timeout, cb) { @@ -197,6 +204,8 @@ class Http2Request extends EventEmitter { this._flushHeaders() } this.stream.setTimeout(timeout, cb) + + return this } removeHeader (headerKey) { @@ -209,6 +218,8 @@ class Http2Request extends EventEmitter { } delete this.requestHeaders[headerKey] + + return this } setHeader (headerKey, headerValue) { @@ -221,6 +232,8 @@ class Http2Request extends EventEmitter { } this.requestHeaders[headerKey] = headerValue + + return this } } @@ -291,22 +304,27 @@ class ResponseProxy extends EventEmitter { pause () { this.reqStream.pause() + return this } resume () { this.reqStream.resume() + return this } pipe (dest) { this.reqStream.pipe(dest) + return dest } setEncoding (encoding) { this.reqStream.setEncoding(encoding) + return this } destroy () { this.reqStream.destroy() + return this } } diff --git a/request.js b/request.js index 1dc91e6a0..1d574ef62 100644 --- a/request.js +++ b/request.js @@ -43,6 +43,7 @@ var toBase64 = helpers.toBase64 var defer = helpers.defer var copy = helpers.copy var version = helpers.version +var SizeTrackerStream = helpers.SizeTrackerStream var globalCookieJar = cookies.jar() var globalPool = {} @@ -1375,6 +1376,8 @@ Request.prototype.onRequestResponse = function (response) { } var responseContent + var downloadSizeTracker = new SizeTrackerStream() + if ((self.gzip || self.brotli) && !noBody(response.statusCode)) { var contentEncoding = response.headers['content-encoding'] || 'identity' contentEncoding = contentEncoding.trim().toLowerCase() @@ -1390,23 +1393,23 @@ Request.prototype.onRequestResponse = function (response) { if (self.gzip && contentEncoding === 'gzip') { responseContent = zlib.createGunzip(zlibOptions) - response.pipe(responseContent) + response.pipe(downloadSizeTracker).pipe(responseContent) } else if (self.gzip && contentEncoding === 'deflate') { responseContent = inflate.createInflate(zlibOptions) - response.pipe(responseContent) + response.pipe(downloadSizeTracker).pipe(responseContent) } else if (self.brotli && contentEncoding === 'br') { responseContent = brotli.createBrotliDecompress() - response.pipe(responseContent) + response.pipe(downloadSizeTracker).pipe(responseContent) } else { // Since previous versions didn't check for Content-Encoding header, // ignore any invalid values to preserve backwards-compatibility if (contentEncoding !== 'identity') { debug('ignoring unrecognized Content-Encoding ' + contentEncoding) } - responseContent = response + responseContent = response.pipe(downloadSizeTracker) } } else { - responseContent = response + responseContent = response.pipe(downloadSizeTracker) } if (self.encoding) { @@ -1436,9 +1439,9 @@ Request.prototype.onRequestResponse = function (response) { // results in some other characters. // For example: If the server intentionally responds with `ð\x9F\x98\x8A` as status message // but if the statusMessageEncoding option is set to `utf8`, then it would get converted to '😊'. - var statusMessage = String(responseContent.statusMessage) + var statusMessage = String(response.statusMessage) if (self.statusMessageEncoding && /[^\w\s-']/.test(statusMessage)) { - responseContent.statusMessage = Buffer.from(statusMessage, 'latin1').toString(self.statusMessageEncoding) + response.statusMessage = Buffer.from(statusMessage, 'latin1').toString(self.statusMessageEncoding) } if (self._paused) { @@ -1483,6 +1486,7 @@ Request.prototype.onRequestResponse = function (response) { self.emit('data', chunk) }) responseContent.once('end', function (chunk) { + self._reqResInfo.response.downloadedBytes = downloadSizeTracker.size self.emit('end', chunk) }) responseContent.on('error', function (error) { diff --git a/tests/server.js b/tests/server.js index 0c0728a8d..adb2417f9 100644 --- a/tests/server.js +++ b/tests/server.js @@ -111,8 +111,6 @@ exports.createPostValidator = function (text, reqContentType) { resp.writeHead(200, { 'content-type': 'text/plain' }) resp.write(r) resp.end() - // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests. - req || req.stream || req.stream.session || req.stream.session.close || req.stream.session.close() }) } return l @@ -133,8 +131,6 @@ exports.createPostJSONValidator = function (value, reqContentType) { resp.writeHead(200, { 'content-type': 'application/json' }) resp.write(r) resp.end() - // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests. - req || req.stream || req.stream.session || req.stream.session.close || req.stream.session.close() }) } return l @@ -145,9 +141,6 @@ exports.createGetResponse = function (text, contentType) { resp.writeHead(200, { 'content-type': contentType }) resp.write(text) resp.end() - - // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests. - req || req.stream || req.stream.session || req.stream.session.close || req.stream.session.close() } return l } @@ -159,8 +152,6 @@ exports.createChunkResponse = function (chunks, contentType) { resp.write(chunk) }) resp.end() - // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests. - req || req.stream || req.stream.session || req.stream.session.close || req.stream.session.close() } return l } diff --git a/tests/test-agent.js b/tests/test-agent.js index 40cdac05f..11fe769ba 100644 --- a/tests/test-agent.js +++ b/tests/test-agent.js @@ -23,16 +23,16 @@ function httpAgent (t, options, req) { var r = (req || request)(options, function (_err, res, body) { t.ok(r.agent instanceof http.Agent, 'is http.Agent') t.equal(r.agent.options.keepAlive, true, 'is keepAlive') - t.equal(Object.keys(r.agent.sockets).length, 1, '1 socket name') - + t.equal(Object.keys(r.agent.freeSockets).length, 1, '1 socket name') var name = (typeof r.agent.getName === 'function') ? r.agent.getName({port: s.port}) : 'localhost:' + s.port // node 0.10- - t.equal(r.agent.sockets[name].length, 1, '1 open socket') - var socket = r.agent.sockets[name][0] + t.equal(r.agent.freeSockets[name].length, 1, '1 open socket') + + var socket = r.agent.freeSockets[name][0] socket.on('close', function () { - t.equal(Object.keys(r.agent.sockets).length, 0, '0 open sockets') + t.equal(Object.keys(r.agent.freeSockets).length, 0, '0 open sockets') t.end() }) socket.end() diff --git a/tests/test-body-http2.js b/tests/test-body-http2.js index ca163acd2..9e5e6f220 100644 --- a/tests/test-body-http2.js +++ b/tests/test-body-http2.js @@ -5,8 +5,10 @@ var request = require('../index') var tape = require('tape') var path = require('path') var fs = require('fs') +var destroyable = require('server-destroy') var s = server.createHttp2Server() +destroyable(s) tape('setup', function (t) { s.listen(0, function () { @@ -154,8 +156,6 @@ addTest('testPutMultipartPostambleCRLF', { tape('testBinaryFile', function (t) { s.on('/', function (req, res) { req.pipe(res) - // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests. - req.stream && req.stream.session && req.stream.session.close && req.stream.session.close() }) request( @@ -178,8 +178,6 @@ tape('testBinaryFile', function (t) { tape('typed array', function (t) { s.on('/', function (req, res) { req.pipe(res) - // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests. - req.stream && req.stream.session && req.stream.session.close && req.stream.session.close() }) var data = new Uint8Array([1, 2, 3]) @@ -203,7 +201,7 @@ tape('typed array', function (t) { }) tape('cleanup', function (t) { - s.close(function () { + s.destroy(function () { t.end() }) }) diff --git a/tests/test-cookies-http2.js b/tests/test-cookies-http2.js index 46ebddf3e..7a399c64d 100644 --- a/tests/test-cookies-http2.js +++ b/tests/test-cookies-http2.js @@ -2,6 +2,7 @@ var request = require('../index') var tape = require('tape') +var destroyable = require('server-destroy') var server = require('./server') var validUrl @@ -9,6 +10,7 @@ var malformedUrl var invalidUrl var s = server.createHttp2Server() +destroyable(s) tape('setup', function (t) { s.listen(0, function () { @@ -19,17 +21,14 @@ tape('setup', function (t) { s.on('/valid', (req, res) => { res.setHeader('set-cookie', 'foo=bar') res.end('okay') - res.stream.session.close() }) s.on('/malformed', (req, res) => { res.setHeader('set-cookie', 'foo') res.end('okay') - res.stream.session.close() }) s.on('/invalid', (req, res) => { res.setHeader('set-cookie', 'foo=bar; Domain=foo.com') res.end('okay') - res.stream.session.close() }) t.end() @@ -137,7 +136,7 @@ tape('custom store', function (t) { }) tape('cleanup', function (t) { - s.close(function () { + s.destroy(function () { t.end() }) }) diff --git a/tests/test-verbose-auto-http2.js b/tests/test-verbose-auto-http2.js index 80b693bea..13d3addd4 100644 --- a/tests/test-verbose-auto-http2.js +++ b/tests/test-verbose-auto-http2.js @@ -77,7 +77,7 @@ tape('HTTP: verbose=true', function (t) { t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) t.deepEqual(Object.keys(debug[0].session.data), ['addresses']) t.equal(debug[0].session.reused, false) - t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) t.notEqual(debug[0].response.headers.length, 0) t.equal(debug[0].response.headers[0].key, 'Date') @@ -116,7 +116,7 @@ tape('HTTP: redirect(HTTPS) + verbose=true', function (t) { t.deepEqual(Object.keys(debug[1].session.data), ['addresses', 'tls']) t.deepEqual(Object.keys(debug[1].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) t.equal(debug[1].session.reused, false) - t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) t.end() }) @@ -144,7 +144,7 @@ tape('HTTPS: verbose=true', function (t) { t.deepEqual(Object.keys(debug[0].session.data), ['addresses', 'tls']) t.deepEqual(Object.keys(debug[0].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) t.equal(debug[0].session.reused, true) - t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) t.end() }) @@ -179,7 +179,7 @@ tape('HTTPS: redirect(HTTP) + verbose=true', function (t) { t.deepEqual(Object.keys(debug[1].session), ['id', 'reused', 'data']) t.deepEqual(Object.keys(debug[1].session.data), ['addresses']) t.equal(debug[1].session.reused, false) - t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) t.end() }) diff --git a/tests/test-verbose-http2.js b/tests/test-verbose-http2.js index 9cb284b38..40f14476a 100644 --- a/tests/test-verbose-http2.js +++ b/tests/test-verbose-http2.js @@ -1,7 +1,9 @@ 'use strict' +var assert = require('assert') var tape = require('tape') var destroyable = require('server-destroy') +var zlib = require('zlib') var server = require('./server') var request = require('../index').defaults({protocolVersion: 'http2'}) @@ -32,6 +34,13 @@ tape('setup', function (t) { res.writeHead(301, { 'location': 'http://localhost:' + plainServer.port + '/' }) res.end() }) + http2Server.on('/gzip', function (req, res) { + res.writeHead(200, { 'content-encoding': 'gzip' }) + zlib.gzip('gzip', function (err, data) { + assert.equal(err, null) + res.end(data) + }) + }) t.end() }) @@ -77,7 +86,7 @@ tape('HTTP: verbose=true', function (t) { t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) t.deepEqual(Object.keys(debug[0].session.data), ['addresses']) t.equal(debug[0].session.reused, false) - t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) t.notEqual(debug[0].response.headers.length, 0) t.equal(debug[0].response.headers[0].key, 'Date') @@ -116,7 +125,7 @@ tape('HTTP: redirect(HTTPS) + verbose=true', function (t) { t.deepEqual(Object.keys(debug[1].session.data), ['addresses', 'tls']) t.deepEqual(Object.keys(debug[1].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) t.equal(debug[1].session.reused, false) - t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) t.end() }) @@ -144,7 +153,36 @@ tape('HTTPS: verbose=true', function (t) { t.deepEqual(Object.keys(debug[0].session.data), ['addresses', 'tls']) t.deepEqual(Object.keys(debug[0].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) t.equal(debug[0].session.reused, true) - t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) + + t.end() + }) +}) + +tape('HTTPS Gzip: verbose=true', function (t) { + var options = { + verbose: true, + strictSSL: false, + time: false, // verbose overrides timing setting + protocolVersion: 'http2' + } + + request('https://localhost:' + http2Server.port + '/gzip', {...options, gzip: true}, function (err, res, body, debug) { + t.equal(err, null) + t.equal(body, 'gzip') + t.equal(Array.isArray(debug), true) + t.equal(debug.length, 1) + + t.equal(typeof res.socket.__SESSION_ID, 'string') + t.equal(typeof res.socket.__SESSION_DATA, 'object') + t.deepEqual(Object.keys(debug[0]), ['request', 'session', 'response', 'timingStart', 'timingStartTimer', 'timings']) + t.deepEqual(Object.keys(debug[0].request), ['method', 'href', 'headers', 'proxy', 'httpVersion']) + t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) + t.deepEqual(Object.keys(debug[0].session.data), ['addresses', 'tls']) + t.deepEqual(Object.keys(debug[0].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) + t.equal(debug[0].session.reused, true) + t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) + t.equal(debug[0].response.downloadedBytes, 24) t.end() }) @@ -179,7 +217,7 @@ tape('HTTPS: redirect(HTTP) + verbose=true', function (t) { t.deepEqual(Object.keys(debug[1].session), ['id', 'reused', 'data']) t.deepEqual(Object.keys(debug[1].session.data), ['addresses']) t.equal(debug[1].session.reused, false) - t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) t.end() }) diff --git a/tests/test-verbose.js b/tests/test-verbose.js index 6941c341f..741ae14d6 100644 --- a/tests/test-verbose.js +++ b/tests/test-verbose.js @@ -1,7 +1,9 @@ 'use strict' +var assert = require('assert') var tape = require('tape') var destroyable = require('server-destroy') +var zlib = require('zlib') var server = require('./server') var request = require('../index') @@ -32,6 +34,13 @@ tape('setup', function (t) { res.writeHead(301, { 'location': 'http://localhost:' + plainServer.port + '/' }) res.end() }) + httpsServer.on('/gzip', function (req, res) { + res.writeHead(200, { 'content-encoding': 'gzip' }) + zlib.gzip('gzip', function (err, data) { + assert.equal(err, null) + res.end(data) + }) + }) t.end() }) @@ -75,7 +84,8 @@ tape('HTTP: verbose=true', function (t) { t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) t.deepEqual(Object.keys(debug[0].session.data), ['addresses']) t.equal(debug[0].session.reused, false) - t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) + t.equal(debug[0].response.downloadedBytes, 5) t.notEqual(debug[0].response.headers.length, 0) t.equal(debug[0].response.headers[0].key, 'Date') @@ -113,7 +123,8 @@ tape('HTTP: redirect(HTTPS) + verbose=true', function (t) { t.deepEqual(Object.keys(debug[1].session.data), ['addresses', 'tls']) t.deepEqual(Object.keys(debug[1].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) t.equal(debug[1].session.reused, false) - t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) + t.equal(debug[1].response.downloadedBytes, 5) t.end() }) @@ -140,7 +151,36 @@ tape('HTTPS: verbose=true', function (t) { t.deepEqual(Object.keys(debug[0].session.data), ['addresses', 'tls']) t.deepEqual(Object.keys(debug[0].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) t.equal(debug[0].session.reused, false) - t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) + t.equal(debug[0].response.downloadedBytes, 5) + + t.end() + }) +}) + +tape('HTTPS Gzip: verbose=true', function (t) { + var options = { + verbose: true, + strictSSL: false, + time: false // verbose overrides timing setting + } + + request('https://localhost:' + httpsServer.port + '/gzip', {...options, gzip: true}, function (err, res, body, debug) { + t.equal(err, null) + t.equal(body, 'gzip') + t.equal(Array.isArray(debug), true) + t.equal(debug.length, 1) + + t.equal(typeof res.socket.__SESSION_ID, 'string') + t.equal(typeof res.socket.__SESSION_DATA, 'object') + t.deepEqual(Object.keys(debug[0]), ['request', 'session', 'response', 'timingStart', 'timingStartTimer', 'timings']) + t.deepEqual(Object.keys(debug[0].request), ['method', 'href', 'headers', 'proxy', 'httpVersion']) + t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) + t.deepEqual(Object.keys(debug[0].session.data), ['addresses', 'tls']) + t.deepEqual(Object.keys(debug[0].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) + t.equal(debug[0].session.reused, false) + t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) + t.equal(debug[0].response.downloadedBytes, 24) t.end() }) @@ -174,7 +214,8 @@ tape('HTTPS: redirect(HTTP) + verbose=true', function (t) { t.deepEqual(Object.keys(debug[1].session), ['id', 'reused', 'data']) t.deepEqual(Object.keys(debug[1].session.data), ['addresses']) t.equal(debug[1].session.reused, false) - t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion']) + t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes']) + t.equal(debug[1].response.downloadedBytes, 5) t.end() })