lib: use utf8 fast path for streaming TextDecoder

PR-URL: #61549
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>

Author: ChALkeR

lib/internal/encoding.js

@@ -28,11 +28,13 @@ const kHandle = Symbol('handle');
 const kFlags = Symbol('flags');
 const kEncoding = Symbol('encoding');
 const kDecoder = Symbol('decoder');
+const kChunk = Symbol('chunk');
 const kFatal = Symbol('kFatal');
 const kUTF8FastPath = Symbol('kUTF8FastPath');
 const kIgnoreBOM = Symbol('kIgnoreBOM');
 
 const { isSinglebyteEncoding, createSinglebyteDecoder } = require('internal/encoding/single-byte');
+const { unfinishedBytesUtf8, mergePrefixUtf8 } = require('internal/encoding/util');
 
 const {
   getConstructorOf,
@@ -447,9 +449,11 @@ class TextDecoder {
     this[kUTF8FastPath] = false;
     this[kHandle] = undefined;
     this[kSingleByte] = undefined; // Does not care about streaming or BOM
+    this[kChunk] = null; // A copy of previous streaming tail or null
 
     if (enc === 'utf-8') {
       this[kUTF8FastPath] = true;
+      this[kBOMSeen] = false;
     } else if (isSinglebyteEncoding(enc)) {
       this[kSingleByte] = createSinglebyteDecoder(enc, this[kFatal]);
     } else {
@@ -458,15 +462,14 @@ class TextDecoder {
   }
 
   #prepareConverter() {
-    if (this[kHandle] !== undefined) return;
     if (hasIntl) {
       let icuEncoding = this[kEncoding];
       if (icuEncoding === 'gbk') icuEncoding = 'gb18030'; // 10.1.1. GBK's decoder is gb18030's decoder
       const handle = icuGetConverter(icuEncoding, this[kFlags]);
       if (handle === undefined)
         throw new ERR_ENCODING_NOT_SUPPORTED(this[kEncoding]);
       this[kHandle] = handle;
-    } else if (this[kEncoding] === 'utf-8' || this[kEncoding] === 'utf-16le') {
+    } else if (this[kEncoding] === 'utf-16le') {
       if (this[kFatal]) throw new ERR_NO_ICU('"fatal" option');
       this[kHandle] = new (lazyStringDecoder())(this[kEncoding]);
       this[kBOMSeen] = false;
@@ -483,11 +486,55 @@ class TextDecoder {
 
     const stream = options?.stream;
     if (this[kUTF8FastPath]) {
-      if (!stream) return decodeUTF8(input, this[kIgnoreBOM], this[kFatal]);
-      this[kUTF8FastPath] = false;
-    }
+      const chunk = this[kChunk];
+      const ignoreBom = this[kIgnoreBOM] || this[kBOMSeen];
+      if (!stream) {
+        this[kBOMSeen] = false;
+        if (!chunk) return decodeUTF8(input, ignoreBom, this[kFatal]);
+      }
+
+      let u = parseInput(input);
+      if (u.length === 0 && stream) return ''; // no state change
+      let prefix;
+      if (chunk) {
+        const merged = mergePrefixUtf8(u, this[kChunk]);
+        if (u.length < 3) {
+          u = merged; // Might be unfinished, but fully consumed old u
+        } else {
+          prefix = merged; // Stops at complete chunk
+          const add = prefix.length - this[kChunk].length;
+          if (add > 0) u = u.subarray(add);
+        }
+
+        this[kChunk] = null;
+      }
 
-    this.#prepareConverter();
+      if (stream) {
+        const trail = unfinishedBytesUtf8(u, u.length);
+        if (trail > 0) {
+          this[kChunk] = new FastBuffer(u.subarray(-trail)); // copy
+          if (!prefix && trail === u.length) return ''; // No further state change
+          u = u.subarray(0, -trail);
+        }
+      }
+
+      try {
+        const res = (prefix ? decodeUTF8(prefix, ignoreBom, this[kFatal]) : '') +
+                    decodeUTF8(u, ignoreBom || prefix, this[kFatal]);
+
+        // "BOM seen" is set on the current decode call only if it did not error,
+        // in "serialize I/O queue" after decoding
+        // We don't get here if we had no complete data to process,
+        // and we don't want BOM processing after that if streaming
+        if (stream) this[kBOMSeen] = true;
+
+        return res;
+      } catch (e) {
+        this[kChunk] = null; // Reset unfinished chunk on errors
+        // The correct way per spec seems to be not destroying the decoder state (aka BOM here) in stream mode
+        throw e;
+      }
+    }
 
     if (hasIntl) {
       const flags = stream ? 0 : CONVERTER_FLAGS_FLUSH;

lib/internal/encoding/util.js

@@ -0,0 +1,72 @@
+// From https://npmjs.com/package/@exodus/bytes
+// Copyright Exodus Movement. Licensed under MIT License.
+
+'use strict';
+
+const {
+  Uint8Array,
+} = primordials;
+
+
+/**
+ * Get a number of last bytes in an Uint8Array `data` ending at `len` that don't
+ * form a codepoint yet, but can be a part of a single codepoint on more data.
+ * @param {Uint8Array} data Uint8Array of potentially UTF-8 bytes
+ * @param {number} len Position to look behind from
+ * @returns {number} Number of unfinished potentially valid UTF-8 bytes ending at position `len`
+ */
+function unfinishedBytesUtf8(data, len) {
+  // 0-3
+  let pos = 0;
+  while (pos < 2 && pos < len && (data[len - pos - 1] & 0xc0) === 0x80) pos++; // Go back 0-2 trailing bytes
+  if (pos === len) return 0; // no space for lead
+  const lead = data[len - pos - 1];
+  if (lead < 0xc2 || lead > 0xf4) return 0; // not a lead
+  if (pos === 0) return 1; // Nothing to recheck, we have only lead, return it. 2-byte must return here
+  if (lead < 0xe0 || (lead < 0xf0 && pos >= 2)) return 0; // 2-byte, or 3-byte or less and we already have 2 trailing
+  const lower = lead === 0xf0 ? 0x90 : lead === 0xe0 ? 0xa0 : 0x80;
+  const upper = lead === 0xf4 ? 0x8f : lead === 0xed ? 0x9f : 0xbf;
+  const next = data[len - pos];
+  return next >= lower && next <= upper ? pos + 1 : 0;
+}
+
+/**
+ * Merge prefix `chunk` with `data` and return new combined prefix.
+ * For data.length < 3, fully consumes data and can return unfinished data,
+ * otherwise returns a prefix with no unfinished bytes
+ * @param {Uint8Array} data Uint8Array of potentially UTF-8 bytes
+ * @param {Uint8Array} chunk Prefix to prepend before `data`
+ * @returns {Uint8Array} If data.length >= 3: an Uint8Array containing `chunk` and a slice of `data`
+ *   so that the result has no unfinished UTF-8 codepoints. If data.length < 3: concat(chunk, data).
+ */
+function mergePrefixUtf8(data, chunk) {
+  if (data.length === 0) return chunk;
+  if (data.length < 3) {
+    // No reason to bruteforce offsets, also it's possible this doesn't yet end the sequence
+    const res = new Uint8Array(data.length + chunk.length);
+    res.set(chunk);
+    res.set(data, chunk.length);
+    return res;
+  }
+
+  // Slice off a small portion of data into prefix chunk so we can decode them separately without extending array size
+  const temp = new Uint8Array(chunk.length + 3); // We have 1-3 bytes and need 1-3 more bytes
+  temp.set(chunk);
+  temp.set(data.subarray(0, 3), chunk.length);
+
+  // Stop at the first offset where unfinished bytes reaches 0 or fits into data
+  // If that doesn't happen (data too short), just concat chunk and data completely (above)
+  for (let i = 1; i <= 3; i++) {
+    const unfinished = unfinishedBytesUtf8(temp, chunk.length + i); // 0-3
+    if (unfinished <= i) {
+      // Always reachable at 3, but we still need 'unfinished' value for it
+      const add = i - unfinished; // 0-3
+      return add > 0 ? temp.subarray(0, chunk.length + add) : chunk;
+    }
+  }
+
+  // Unreachable
+  return null;
+}
+
+module.exports = { unfinishedBytesUtf8, mergePrefixUtf8 };

test/parallel/test-bootstrap-modules.js

@@ -90,6 +90,7 @@ expected.beforePreExec = new Set([
   'Internal Binding fs',
   'NativeModule internal/encoding',
   'NativeModule internal/encoding/single-byte',
+  'NativeModule internal/encoding/util',
   'NativeModule internal/blob',
   'NativeModule internal/fs/utils',
   'NativeModule fs',

test/parallel/test-whatwg-encoding-custom-textdecoder.js

@@ -80,20 +80,8 @@ assert(TextDecoder);
 
 ['unicode-1-1-utf-8', 'utf8', 'utf-8'].forEach((i) => {
   const dec = new TextDecoder(i, { fatal: true });
-  if (common.hasIntl) {
-    dec.decode(buf.slice(0, 8), { stream: true });
-    dec.decode(buf.slice(8));
-  } else {
-    assert.throws(
-      () => {
-        dec.decode(buf.slice(0, 8), { stream: true });
-      },
-      {
-        code: 'ERR_NO_ICU',
-        name: 'TypeError',
-        message: '"fatal" option is not supported on Node.js compiled without ICU'
-      });
-  }
+  dec.decode(buf.slice(0, 8), { stream: true });
+  dec.decode(buf.slice(8));
 });
 
 // Test TextDecoder, label undefined, options null
@@ -122,33 +110,16 @@ if (common.hasIntl) {
 // Test TextDecoder inspect with hidden fields
 {
   const dec = new TextDecoder('utf-8', { ignoreBOM: true });
-  if (common.hasIntl) {
-    assert.strictEqual(
-      util.inspect(dec, { showHidden: true }),
-      'TextDecoder {\n' +
-      '  encoding: \'utf-8\',\n' +
-      '  fatal: false,\n' +
-      '  ignoreBOM: true,\n' +
-      '  Symbol(flags): 4,\n' +
-      '  Symbol(handle): undefined\n' +
-      '}'
-    );
-  } else {
-    dec.decode(Uint8Array.of(0), { stream: true });
-    assert.strictEqual(
-      util.inspect(dec, { showHidden: true }),
-      'TextDecoder {\n' +
-      "  encoding: 'utf-8',\n" +
-      '  fatal: false,\n' +
-      '  ignoreBOM: true,\n' +
-      '  Symbol(flags): 4,\n' +
-      '  Symbol(handle): StringDecoder {\n' +
-      "    encoding: 'utf8',\n" +
-      '    Symbol(kNativeDecoder): <Buffer 00 00 00 00 00 00 01>\n' +
-      '  }\n' +
-      '}'
-    );
-  }
+  assert.strictEqual(
+    util.inspect(dec, { showHidden: true }),
+    'TextDecoder {\n' +
+    '  encoding: \'utf-8\',\n' +
+    '  fatal: false,\n' +
+    '  ignoreBOM: true,\n' +
+    '  Symbol(flags): 4,\n' +
+    '  Symbol(handle): undefined\n' +
+    '}'
+  );
 }