-
Notifications
You must be signed in to change notification settings - Fork 30.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Bug report] fs.createReadStream set highWaterMark, the same file get different result #35465
Comments
when bytesRead === 0 why poolFragments.push(thisPool.slice(alignedStart, alignedEnd)); node-v12.2.0/lib/internal/fs/streams.js ReadStream.prototype._read = function(n) {
if (typeof this.fd !== 'number') {
return this.once('open', function() {
this._read(n);
});
}
if (this.destroyed)
return;
if (!pool || pool.length - pool.used < kMinPoolSpace) {
// Discard the old pool.
allocNewPool(this.readableHighWaterMark);
}
// Grab another reference to the pool in the case that while we're
// in the thread pool another read() finishes up the pool, and
// allocates a new one.
const thisPool = pool;
let toRead = Math.min(pool.length - pool.used, n);
const start = pool.used;
if (this.pos !== undefined)
toRead = Math.min(this.end - this.pos + 1, toRead);
else
toRead = Math.min(this.end - this.bytesRead + 1, toRead);
// Already read everything we were supposed to read!
// treat as EOF.
if (toRead <= 0)
return this.push(null);
// the actual read.
fs.read(this.fd, pool, pool.used, toRead, this.pos, (er, bytesRead) => {
if (er) {
if (this.autoClose) {
this.destroy();
}
this.emit('error', er);
} else {
let b = null;
// Now that we know how much data we have actually read, re-wind the
// 'used' field if we can, and otherwise allow the remainder of our
// reservation to be used as a new pool later.
if (start + toRead === thisPool.used && thisPool === pool) {
console.log('newUsed')
const newUsed = thisPool.used + bytesRead - toRead;
thisPool.used = roundUpToMultipleOf8(newUsed);
} else {
console.log('alignedEnd')
console.log('bytesRead: ' + bytesRead)
// Round down to the next lowest multiple of 8 to ensure the new pool
// fragment start and end positions are aligned to an 8 byte boundary.
const alignedEnd = (start + toRead) & ~7;
const alignedStart = roundUpToMultipleOf8(start + bytesRead);
if (alignedEnd - alignedStart >= kMinPoolSpace) {
console.log('start: ' + start)
console.log('toRead: ' + toRead)
console.log('alignedEnd: ' + alignedEnd)
console.log('alignedStart: ' + alignedStart)
poolFragments.push(thisPool.slice(alignedStart, alignedEnd));
}
}
if (bytesRead > 0) {
this.bytesRead += bytesRead;
b = thisPool.slice(start, start + bytesRead);
}
this.push(b);
}
});
|
Works fine if (start + toRead === thisPool.used && thisPool === pool) {
const newUsed = thisPool.used + bytesRead - toRead;
thisPool.used = roundUpToMultipleOf8(newUsed);
- } else {
+ } else if (bytesRead > 0) { |
@Zclhlmgqzc PR welcome! Do you have the same issue on Node 15/master? |
Hi, is there an update on this? Do you think it'll make it into 12.x or 14.x? I don't have a ton of extra time, but if you give me some direction, I can take a swing at fixing it. |
Closing as v14.x is EoL. |
12.2.0-12.18.4-14.13.0
Windows10,
Linux version 3.10.0-957.el7.x86_64 (mockbuild@kbuilder.bsys.centos.org) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-36) (GCC) ) deps: update openssl to 1.0.1j #1 SMP Thu Nov 8 23:39:32 UTC 2018
What steps will reproduce the bug?
test-202.js
test-file-202
(10 * 20 + /r/n)
How often does it reproduce? Is there a required condition?
Always with node v12.2.0-v12.18.4-14.13.0 and highWaterMark > 128
Works fine in node v10.22.1 v12.1.0 or highWaterMark <= 128
What is the expected behavior?
the same result
What do you see instead?
Additional information
v12.2.0
fs: align fs.ReadStream buffer pool writes to 8-byte boundary
#24838
The text was updated successfully, but these errors were encountered: