diff --git a/README.md b/README.md index acb72a5..b87c796 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,10 @@ is directly passed as a [Transform](https://nodejs.org/api/stream.html#stream_new_stream_transform_options) option. -Additionally, the `.maxLength` option is implemented, which will make the split stream throw an error -if the buffer size exceeds `.maxLength`. +Additionally, the `.maxLength` and `.skipOverflow` options are implemented, which set limits on the internal +buffer size and the stream's behavior when the limit is exceeded. There is no limit unless `maxLength` is set. When +the internal buffer size exceeds `maxLength`, the stream emits an error by default. You may also set `skipOverflow` to +true to suppress the error and instead skip past any lines that cause the internal buffer to exceed `maxLength`. Calling `.destroy` will make the stream emit `close`. Use this to perform cleanup logic diff --git a/index.js b/index.js index 57d76a4..bec5100 100644 --- a/index.js +++ b/index.js @@ -22,19 +22,30 @@ const kLast = Symbol('last') const kDecoder = Symbol('decoder') function transform (chunk, enc, cb) { - this[kLast] += this[kDecoder].write(chunk) - if (this[kLast].length > this.maxLength) { - return cb(new Error('maximum buffer reached')) + var list + if (this.overflow) { // Line buffer is full. Skip to start of next line. + var buf = this[kDecoder].write(chunk) + list = buf.split(this.matcher) + + if (list.length === 1) return cb() // Line ending not found. Discard entire chunk. + + // Line ending found. Discard trailing fragment of previous line and reset overflow state. + list.shift() + this.overflow = false + } else { + this[kLast] += this[kDecoder].write(chunk) + list = this[kLast].split(this.matcher) } - var list = this[kLast].split(this.matcher) - this[kLast] = list.pop() for (var i = 0; i < list.length; i++) { push(this, this.mapper(list[i])) } + this.overflow = this[kLast].length > this.maxLength + if (this.overflow && !this.skipOverflow) return cb(new Error('maximum buffer reached')) + cb() } @@ -103,6 +114,8 @@ function split (matcher, mapper, options) { stream.matcher = matcher stream.mapper = mapper stream.maxLength = options.maxLength + stream.skipOverflow = options.skipOverflow + stream.overflow = false return stream } diff --git a/test.js b/test.js index 9715dfd..219d67f 100644 --- a/test.js +++ b/test.js @@ -302,3 +302,32 @@ test('readable highWaterMark', function (t) { t.equal(input._readableState.highWaterMark, 16) t.end() }) + +test('maxLength < chunk size', function (t) { + t.plan(2) + + var input = split({ maxLength: 2 }) + + input.pipe(strcb(function (err, list) { + t.error(err) + t.deepEqual(list, ['a', 'b']) + })) + + input.end('a\nb') +}) + +test('maximum buffer limit w/skip', function (t) { + t.plan(2) + + var input = split({ maxLength: 2, skipOverflow: true }) + + input.pipe(strcb(function (err, list) { + t.error(err) + t.deepEqual(list, ['a', 'b', 'c']) + })) + + input.write('a\n123') + input.write('456') + input.write('789\nb\nc') + input.end() +})