From d05b777fbbd0f2f219d1065330079529ee56b076 Mon Sep 17 00:00:00 2001
From: Robert Nagy <ronagy@icloud.com>
Date: Sun, 20 Oct 2019 10:13:57 +0200
Subject: [PATCH] stream: increase MAX_HWM

MAX_HWM was added in 9208c89 where the highwatermark was changed to
always increase in steps of highest power of 2 to prevent increasing
hwm excessivly in tiny amounts.

Why a limit was added on the highwatermark is unclear but breaks
existing usage where a larger read size is used. The invariant for
read(n) is that a buffer of size n is always returned. Considering
a maximum ceiling on the buffer size breaks this invariant.

This PR significantly increases the limit to make it less likely to
break the previous invariant and also documents the limit.

Fixes: https://github.com/nodejs/node/issues/29933
---
 lib/_stream_readable.js                  | 25 ++++------------------
 test/parallel/test-readable-large-hwm.js | 27 ++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 21 deletions(-)
 create mode 100644 test/parallel/test-readable-large-hwm.js

diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js
index 5d2300cf891bca..ba5b9b6986c3a9 100644
--- a/lib/_stream_readable.js
+++ b/lib/_stream_readable.js
@@ -374,25 +374,6 @@ Readable.prototype.setEncoding = function(enc) {
   return this;
 };
 
-// Don't raise the hwm > 8MB
-const MAX_HWM = 0x800000;
-function computeNewHighWaterMark(n) {
-  if (n >= MAX_HWM) {
-    n = MAX_HWM;
-  } else {
-    // Get the next highest power of 2 to prevent increasing hwm excessively in
-    // tiny amounts
-    n--;
-    n |= n >>> 1;
-    n |= n >>> 2;
-    n |= n >>> 4;
-    n |= n >>> 8;
-    n |= n >>> 16;
-    n++;
-  }
-  return n;
-}
-
 // This function is designed to be inlinable, so please take care when making
 // changes to the function body.
 function howMuchToRead(n, state) {
@@ -425,9 +406,11 @@ Readable.prototype.read = function(n) {
   const state = this._readableState;
   const nOrig = n;
 
-  // If we're asking for more than the current hwm, then raise the hwm.
+  // If we're asking for more than the current hwm, then raise the hwm to
+  // the next highest power of 2 to prevent increasing hwm excessively in
+  // tiny amounts.
   if (n > state.highWaterMark)
-    state.highWaterMark = computeNewHighWaterMark(n);
+    state.highWaterMark = Math.pow(2, Math.ceil(Math.log(n) / Math.log(2)));
 
   if (n !== 0)
     state.emittedReadable = false;
diff --git a/test/parallel/test-readable-large-hwm.js b/test/parallel/test-readable-large-hwm.js
new file mode 100644
index 00000000000000..d5bf25bc0e61c1
--- /dev/null
+++ b/test/parallel/test-readable-large-hwm.js
@@ -0,0 +1,27 @@
+'use strict';
+const common = require('../common');
+const { Readable } = require('stream');
+
+// Make sure that readable completes
+// even when reading larger buffer.
+const bufferSize = 10 * 1024 * 1024;
+let n = 0;
+const r = new Readable({
+  read() {
+    // Try to fill readable buffer piece by piece.
+    r.push(Buffer.alloc(bufferSize / 10));
+
+    if (n++ > 10) {
+      r.push(null);
+    }
+  }
+});
+
+r.on('readable', () => {
+  while (true) {
+    const ret = r.read(bufferSize);
+    if (ret === null)
+      break;
+  }
+});
+r.on('end', common.mustCall());