From 0d4942836a3a3ca9085df087f2509216075197e4 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 3 Apr 2019 02:58:31 +0200 Subject: [PATCH 1/4] fs: improve readFile performance This increases the maximum buffer size per read to 256kb when using `fs.readFile`. This is important to improve the read performance for bigger files. Refs: https://github.com/nodejs/node/issues/25741 --- lib/internal/fs/read_file_context.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/internal/fs/read_file_context.js b/lib/internal/fs/read_file_context.js index 4961ee820a7e84..5397049b809149 100644 --- a/lib/internal/fs/read_file_context.js +++ b/lib/internal/fs/read_file_context.js @@ -79,7 +79,8 @@ class ReadFileContext { } else { buffer = this.buffer; offset = this.pos; - length = Math.min(kReadFileBufferLength, this.size - this.pos); + // Read up to 256kb. + length = Math.min(256 * 1024, this.size - this.pos); } const req = new FSReqCallback(); From 8c248957b0dfb84af03f0b0b32efc3c55887d2bc Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Thu, 4 Apr 2019 19:10:23 +0200 Subject: [PATCH 2/4] fixup! fs: improve readFile performance --- lib/fs.js | 12 +++++------ lib/internal/fs/read_file_context.js | 31 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/lib/fs.js b/lib/fs.js index b27b7f8916657a..b1f8e2ec7053ef 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -264,19 +264,17 @@ function readFileAfterStat(err, stats) { const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0; - if (size === 0) { - context.buffers = []; - context.read(); - return; - } - if (size > kMaxLength) { err = new ERR_FS_FILE_TOO_LARGE(size); return context.close(err); } try { - context.buffer = Buffer.allocUnsafeSlow(size); + if (size === 0) { + context.buffers = []; + } else { + context.buffer = Buffer.allocUnsafeSlow(size); + } } catch (err) { return context.close(err); } diff --git a/lib/internal/fs/read_file_context.js b/lib/internal/fs/read_file_context.js index 5397049b809149..c1bc5f35a0be15 100644 --- a/lib/internal/fs/read_file_context.js +++ b/lib/internal/fs/read_file_context.js @@ -4,7 +4,8 @@ const { Buffer } = require('buffer'); const { FSReqCallback, close, read } = internalBinding('fs'); -const kReadFileBufferLength = 8 * 1024; +const kReadFileUnknownBufferLength = 8 * 1024; +const kReadFileBufferLength = 256 * 1024; function readFileAfterRead(err, bytesRead) { const context = this.context; @@ -12,19 +13,17 @@ function readFileAfterRead(err, bytesRead) { if (err) return context.close(err); - if (bytesRead === 0) - return context.close(); - context.pos += bytesRead; - if (context.size !== 0) { - if (context.pos === context.size) - context.close(); - else - context.read(); + if (context.pos === context.size || bytesRead === 0) { + context.close(); } else { - // Unknown size, just read until we don't get bytes. - context.buffers.push(context.buffer.slice(0, bytesRead)); + if (context.size === 0) { + // Unknown size, just read until we don't get bytes. + const buffer = bytesRead === kReadFileUnknownBufferLength ? + context.buffer : context.buffer.slice(0, bytesRead); + context.buffers.push(buffer); + } context.read(); } } @@ -58,7 +57,7 @@ class ReadFileContext { constructor(callback, encoding) { this.fd = undefined; this.isUserFd = undefined; - this.size = undefined; + this.size = 0; this.callback = callback; this.buffers = null; this.buffer = null; @@ -73,14 +72,14 @@ class ReadFileContext { let length; if (this.size === 0) { - buffer = this.buffer = Buffer.allocUnsafeSlow(kReadFileBufferLength); + buffer = Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength); + this.buffer = buffer; offset = 0; - length = kReadFileBufferLength; + length = kReadFileUnknownBufferLength; } else { buffer = this.buffer; offset = this.pos; - // Read up to 256kb. - length = Math.min(256 * 1024, this.size - this.pos); + length = Math.min(kReadFileBufferLength, this.size - this.pos); } const req = new FSReqCallback(); From b64266e5732891c9518f2f31421f70e245fa71e0 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Thu, 4 Apr 2019 22:45:14 +0200 Subject: [PATCH 3/4] fixup! fs: improve readFile performance --- lib/internal/fs/read_file_context.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/internal/fs/read_file_context.js b/lib/internal/fs/read_file_context.js index c1bc5f35a0be15..59dd45d49ce372 100644 --- a/lib/internal/fs/read_file_context.js +++ b/lib/internal/fs/read_file_context.js @@ -4,8 +4,8 @@ const { Buffer } = require('buffer'); const { FSReqCallback, close, read } = internalBinding('fs'); -const kReadFileUnknownBufferLength = 8 * 1024; -const kReadFileBufferLength = 256 * 1024; +const kReadFileUnknownBufferLength = 16 * 1024; +const kReadFileBufferLength = 512 * 1024; function readFileAfterRead(err, bytesRead) { const context = this.context; @@ -73,9 +73,9 @@ class ReadFileContext { if (this.size === 0) { buffer = Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength); - this.buffer = buffer; offset = 0; length = kReadFileUnknownBufferLength; + this.buffer = buffer; } else { buffer = this.buffer; offset = this.pos; From 36774fe995267e69717a910f1f038346fa5fdd61 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Sat, 6 Apr 2019 00:43:58 +0200 Subject: [PATCH 4/4] fixup! fs: improve readFile performance --- lib/internal/fs/read_file_context.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/internal/fs/read_file_context.js b/lib/internal/fs/read_file_context.js index 59dd45d49ce372..4dec92f19e6cd7 100644 --- a/lib/internal/fs/read_file_context.js +++ b/lib/internal/fs/read_file_context.js @@ -4,7 +4,13 @@ const { Buffer } = require('buffer'); const { FSReqCallback, close, read } = internalBinding('fs'); -const kReadFileUnknownBufferLength = 16 * 1024; +// Use 64kb in case the file type is not a regular file and thus do not know the +// actual file size. Increasing the value further results in more frequent over +// allocation for small files and consumes CPU time and memory that should be +// used else wise. +// Use up to 512kb per read otherwise to partition reading big files to prevent +// blocking other threads in case the available threads are all in use. +const kReadFileUnknownBufferLength = 64 * 1024; const kReadFileBufferLength = 512 * 1024; function readFileAfterRead(err, bytesRead) {