stream: update TextEncoderStream to align the latest spec

PR-URL: #44101 Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com> Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
nodejs · Aug 22, 2022 · 2f904bc · 2f904bc
1 parent 123b2d6
commit 2f904bc
Show file tree

Hide file tree

Showing 2 changed files with 66 additions and 8 deletions.
diff --git a/lib/internal/webstreams/encoding.js b/lib/internal/webstreams/encoding.js
@@ -2,7 +2,10 @@
 
 const {
   ObjectDefineProperties,
+  String,
+  StringPrototypeCharCodeAt,
   Symbol,
+  Uint8Array,
 } = primordials;
 
 const {
@@ -31,6 +34,7 @@ const {
 const kHandle = Symbol('kHandle');
 const kTransform = Symbol('kTransform');
 const kType = Symbol('kType');
+const kPendingHighSurrogate = Symbol('kPendingHighSurrogate');
 
 /**
  * @typedef {import('./readablestream').ReadableStream} ReadableStream
@@ -49,19 +53,46 @@ function isTextDecoderStream(value) {
 
 class TextEncoderStream {
   constructor() {
+    this[kPendingHighSurrogate] = null;
     this[kType] = 'TextEncoderStream';
     this[kHandle] = new TextEncoder();
     this[kTransform] = new TransformStream({
       transform: (chunk, controller) => {
-        const value = this[kHandle].encode(chunk);
-        if (value)
+        // https://encoding.spec.whatwg.org/#encode-and-enqueue-a-chunk
+        chunk = String(chunk);
+        let finalChunk = '';
+        for (let i = 0; i < chunk.length; i++) {
+          const item = chunk[i];
+          const codeUnit = StringPrototypeCharCodeAt(item, 0);
+          if (this[kPendingHighSurrogate] !== null) {
+            const highSurrogate = this[kPendingHighSurrogate];
+            this[kPendingHighSurrogate] = null;
+            if (0xDC00 <= codeUnit && codeUnit <= 0xDFFF) {
+              finalChunk += highSurrogate + item;
+              continue;
+            }
+            finalChunk += '\uFFFD';
+          }
+          if (0xD800 <= codeUnit && codeUnit <= 0xDBFF) {
+            this[kPendingHighSurrogate] = item;
+            continue;
+          }
+          if (0xDC00 <= codeUnit && codeUnit <= 0xDFFF) {
+            finalChunk += '\uFFFD';
+            continue;
+          }
+          finalChunk += item;
+        }
+        if (finalChunk) {
+          const value = this[kHandle].encode(finalChunk);
           controller.enqueue(value);
+        }
       },
       flush: (controller) => {
-        const value = this[kHandle].encode();
-        if (value.byteLength > 0)
-          controller.enqueue(value);
-        controller.terminate();
+        // https://encoding.spec.whatwg.org/#encode-and-flush
+        if (this[kPendingHighSurrogate] !== null) {
+          controller.enqueue(new Uint8Array([0xEF, 0xBF, 0xBD]));
+        }
       },
     });
   }

diff --git a/test/wpt/status/encoding.json b/test/wpt/status/encoding.json
@@ -48,8 +48,35 @@
   "unsupported-encodings.any.js": {
     "skip": "decoding-helpers.js needs XMLHttpRequest"
   },
-  "streams/*.js": {
-    "fail": "No implementation of TextDecoderStream and TextEncoderStream"
+  "streams/decode-ignore-bom.any.js": {
+    "requires": ["small-icu"]
+  },
+  "streams/realms.window.js": {
+    "skip": "window is not defined"
+  },
+  "streams/decode-attributes.any.js": {
+    "requires": ["full-icu"]
+  },
+  "streams/decode-incomplete-input.any.js": {
+    "requires": ["small-icu"]
+  },
+  "streams/decode-utf8.any.js": {
+    "requires": ["small-icu"],
+    "fail": {
+      "unexpected": [
+        "promise_test: Unhandled rejection with value: object 'TypeError: Cannot perform Construct on a detached ArrayBuffer'"
+      ]
+    }
+  },
+  "streams/decode-bad-chunks.any.js": {
+    "fail": {
+      "unexpected": [
+        "assert_unreached: Should have rejected: write should reject Reached unreachable code"
+      ]
+    }
+  },
+  "streams/decode-non-utf8.any.js": {
+    "requires": ["full-icu"]
   },
   "encodeInto.any.js": {
     "requires": ["small-icu"]