How do I get the response size of a request?except by Content-Length

[drdevelop]

Steps to reproduce

Tell us about your environment:

• Puppeteer version:
• Platform / OS version:
• URLs (if applicable):
• Node.js version:

What steps will reproduce the problem?

Please include code that reproduces the issue.

What is the expected result?

What happens instead?

[vsemozhetbyt]

Via response.buffer()? But this will be decoded (unzipped) body. Compare:

'use strict';

const puppeteer = require('puppeteer');

(async function main() {
  try {
    const browser = await puppeteer.launch();
    const [page] = await browser.pages();

    const response1 = await page.goto('https://example.org/');
    console.log(
      response1.headers()['content-length'],
      (await response1.buffer()).length
    );

    page.setExtraHTTPHeaders({ 'accept-encoding': 'identity'});

    const response2 = await page.goto('https://example.org/');
    console.log(
      response2.headers()['content-length'],
      (await response2.buffer()).length
    );

    await browser.close();
  } catch (err) {
    console.error(err);
  }
})();

606 1270
1270 1270

[drdevelop]

No, I mean, can I get the size of each response body, like the Google console, regardless of whether the Content-Length field is in the response header returned by the back end, because the gzip-compressed response header has no Content-Length field

[Ivlyth]

chunked response has no Content-Length header .

response in CDP has encodedDataLength but puppeteer don't save it.

[entrptaher]

Another way is to use CDP.

// Monitor using CDP
const devToolsResponses = new Map();
const devTools = await page.target().createCDPSession();
await devTools.send("Network.enable");

devTools.on("Network.responseReceived", event => {
  devToolsResponses.set(event.requestId, event.response);
});

devTools.on("Network.loadingFinished", event => {
  const response = devToolsResponses.get(event.requestId);
  const encodedBodyLength =
    event.encodedDataLength - response.headersText.length;
  console.log(`${encodedBodyLength} bytes for ${response.url}`);
});

However be careful using this. await page.setRequestInterception(true); will screw this up right this moment.

Such questions are more suitable for Stackoverflow since it's not an issue/feature request actually, here is a similar question.

[Ivlyth]

currently i'm using the code blew to hook into puppeteer to add rawResponse from CDP to puppeteer's Response.

the puppeteer version i use is 1.8.0

const {NetworkManager, Response} = require('puppeteer/lib/NetworkManager');

NetworkManager.prototype._onResponseReceived = function (event) {
    const request = this._requestIdToRequest.get(event.requestId);
    // FileUpload sends a response without a matching request.
    if (!request)
        return;
    const response = new Response(this._client, request, event.response);
    response.rawResponse = event.response;   // <<<<<<<<< this is what i add
    request._response = response;
    this.emit(NetworkManager.Events.Response, response);
};

NetworkManager.prototype._handleRequestRedirect = function (request, responsePayload) {
    const response = new Response(this._client, request, responsePayload);
    response.rawResponse = responsePayload;   // <<<<<<<<< this is what i add
    request._response = response;
    request._redirectChain.push(request);
    response._bodyLoadedPromiseFulfill.call(null, new Error('Response body is unavailable for redirect responses'));
    this._requestIdToRequest.delete(request._requestId);
    this._attemptedAuthentications.delete(request._interceptionId);
    this.emit(NetworkManager.Events.Response, response);
    this.emit(NetworkManager.Events.RequestFinished, request);
};

after then, when you get a response from puppeteer, you can use response.rawResponse. encodedDataLength to get response size

[entrptaher]

@MythRen Very interesting approach.

[entrptaher]

More to add to this problem, here is a target url.

URL: https://fonts.googleapis.com/css?family=YT%20Sans%3A300%2C500%2C700

The answers at #3372 (comment) and #3372 (comment), seems to not work at all when page.setRequestInterception is true. The target url does not have this header so we cannot use that.

Is this a upstream bug @aslushnikov ?

[stale]

We're marking this issue as unconfirmed because it has not had recent activity and we weren't able to confirm it yet. It will be closed if no further activity occurs within the next 30 days.

[stale]

We are closing this issue. If the issue still persists in the latest version of Puppeteer, please reopen the issue and update the description. We will try our best to accomodate it!

[clouedoc]

My approach was to edit Puppeteer's code directly with patch-package:

Patch

diff --git a/node_modules/puppeteer-core/lib/cjs/puppeteer/common/HTTPResponse.js b/node_modules/puppeteer-core/lib/cjs/puppeteer/common/HTTPResponse.js
index b7ce222..e29119c 100644
--- a/node_modules/puppeteer-core/lib/cjs/puppeteer/common/HTTPResponse.js
+++ b/node_modules/puppeteer-core/lib/cjs/puppeteer/common/HTTPResponse.js
@@ -57,6 +57,8 @@ class HTTPResponse extends HTTPResponse_js_1.HTTPResponse {
             ? new SecurityDetails_js_1.SecurityDetails(responsePayload.securityDetails)
             : null, "f");
         __classPrivateFieldSet(this, _HTTPResponse_timing, responsePayload.timing || null, "f");
+      this._responsePayload = responsePayload;
+      this._extraInfo = extraInfo;
     }
     _resolveBody(err) {
         if (err) {
diff --git a/node_modules/puppeteer-core/lib/esm/puppeteer/common/HTTPResponse.js b/node_modules/puppeteer-core/lib/esm/puppeteer/common/HTTPResponse.js
index cd455ca..919b97b 100644
--- a/node_modules/puppeteer-core/lib/esm/puppeteer/common/HTTPResponse.js
+++ b/node_modules/puppeteer-core/lib/esm/puppeteer/common/HTTPResponse.js
@@ -54,6 +54,8 @@ export class HTTPResponse extends BaseHTTPResponse {
             ? new SecurityDetails(responsePayload.securityDetails)
             : null, "f");
         __classPrivateFieldSet(this, _HTTPResponse_timing, responsePayload.timing || null, "f");
+        this._responsePayload = responsePayload;
+        this._extraInfo = extraInfo;
     }
     _resolveBody(err) {
         if (err) {

Typings

// filename: puppeteer.d.ts
import {} from 'puppeteer';

interface IHTTPResponseAdditions {
  _responsePayload: {
    encodedDataLength: number;
  };
}

declare module 'puppeteer' {
  interface HTTPResponse extends IHTTPResponseAdditions {}
}

declare module 'puppeteer-core' {
  interface HTTPResponse extends IHTTPResponseAdditions {}
}

Updated HTTPResponse

There is now a _responsePayload property as well as an _extraInfos one:

HTTPResponse {
  _responsePayload: {
    url: 'https://m.media-amazon.com/images/G/08/x-locale/common/transparent-pixel._CB485934981_.gif',
    status: 200,
    statusText: 'OK',
    headers: {
      accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
      referer: 'https://www.amazon.fr/',
      'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.126 Safari/537.36',
      'content-length': '5982'
    },
    mimeType: 'image/gif',
    connectionReused: false,
    connectionId: 0,
    fromDiskCache: false,
    fromServiceWorker: false,
    fromPrefetchCache: false,
    encodedDataLength: 274,
    timing: {
      requestTime: 212763.45712,
      proxyStart: -1,
      proxyEnd: -1,
      dnsStart: -1,
      dnsEnd: -1,
      connectStart: -1,
      connectEnd: -1,
      sslStart: -1,
      sslEnd: -1,
      workerStart: -1,
      workerReady: -1,
      workerFetchStart: -1,
      workerRespondWithSettled: -1,
      sendStart: -1,
      sendEnd: -1,
      pushStart: 0,
      pushEnd: 0,
      receiveHeadersEnd: 3.534
    },
    responseTime: 1685116891815.947,
    protocol: 'http/1.1',
    alternateProtocolUsage: 'alternativeJobWonWithoutRace',
    securityState: 'unknown'
  },
  _extraInfo: null
}