test: update WPT encoding tests

nodejs · Jul 23, 2022 · 1fd63f8 · 1fd63f8
1 parent 7a92662
commit 1fd63f8
Show file tree

Hide file tree

Showing 15 changed files with 252 additions and 27 deletions.
diff --git a/test/fixtures/wpt/README.md b/test/fixtures/wpt/README.md
@@ -14,7 +14,7 @@ Last update:
 - console: https://github.com/web-platform-tests/wpt/tree/3b1f72e99a/console
 - dom/abort: https://github.com/web-platform-tests/wpt/tree/c49cafb491/dom/abort
 - dom/events: https://github.com/web-platform-tests/wpt/tree/f8821adb28/dom/events
-- encoding: https://github.com/web-platform-tests/wpt/tree/35f70910d3/encoding
+- encoding: https://github.com/web-platform-tests/wpt/tree/c1b24fce6e/encoding
 - FileAPI: https://github.com/web-platform-tests/wpt/tree/3b279420d4/FileAPI
 - hr-time: https://github.com/web-platform-tests/wpt/tree/9910784394/hr-time
 - html/webappapis/atob: https://github.com/web-platform-tests/wpt/tree/f267e1dca6/html/webappapis/atob

diff --git a/test/fixtures/wpt/encoding/encodeInto.any.js b/test/fixtures/wpt/encoding/encodeInto.any.js
@@ -120,19 +120,22 @@
   });
 });
 
-[DataView,
- Int8Array,
- Int16Array,
- Int32Array,
- Uint16Array,
- Uint32Array,
- Uint8ClampedArray,
- Float32Array,
- Float64Array].forEach(view => {
+["DataView",
+ "Int8Array",
+ "Int16Array",
+ "Int32Array",
+ "Uint16Array",
+ "Uint32Array",
+ "Uint8ClampedArray",
+ "BigInt64Array",
+ "BigUint64Array",
+ "Float32Array",
+ "Float64Array"].forEach(type => {
   ["ArrayBuffer", "SharedArrayBuffer"].forEach((arrayBufferOrSharedArrayBuffer) => {
     test(() => {
-      assert_throws_js(TypeError, () => new TextEncoder().encodeInto("", new view(createBuffer(arrayBufferOrSharedArrayBuffer, 0))));
-    }, "Invalid encodeInto() destination: " + view.name + ", backed by: " + arrayBufferOrSharedArrayBuffer);
+      const viewInstance = new self[type](createBuffer(arrayBufferOrSharedArrayBuffer, 0));
+      assert_throws_js(TypeError, () => new TextEncoder().encodeInto("", viewInstance));
+    }, "Invalid encodeInto() destination: " + type + ", backed by: " + arrayBufferOrSharedArrayBuffer);
   });
 });
 

diff --git a/test/fixtures/wpt/encoding/encodeInto.any.js.headers b/test/fixtures/wpt/encoding/encodeInto.any.js.headers
@@ -0,0 +1,2 @@
+Cross-Origin-Opener-Policy: same-origin
+Cross-Origin-Embedder-Policy: require-corp
diff --git a/test/fixtures/wpt/encoding/idlharness-shadowrealm.window.js b/test/fixtures/wpt/encoding/idlharness-shadowrealm.window.js
@@ -0,0 +1,2 @@
+// META: script=/resources/idlharness-shadowrealm.js
+idl_test_shadowrealm(["encoding"], ["streams"]);
diff --git a/test/fixtures/wpt/encoding/sharedarraybuffer.https.html b/test/fixtures/wpt/encoding/sharedarraybuffer.https.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+
+test(() => {
+  const decoder = new TextDecoder('utf-8');
+  const buffer = new SharedArrayBuffer(4);
+  assert_throws_js(TypeError, () => {
+    decoder.decode(new Uint8Array(buffer));
+  }, 'constructing TextDecoder with SharedArrayBuffer view should throw');
+}, 'decoding SharedArrayBuffer');
+
+</script>
diff --git a/test/fixtures/wpt/encoding/sharedarraybuffer.https.html.headers b/test/fixtures/wpt/encoding/sharedarraybuffer.https.html.headers
@@ -0,0 +1,2 @@
+Cross-Origin-Opener-Policy:same-origin
+Cross-Origin-Embedder-Policy:require-corp
diff --git a/test/fixtures/wpt/encoding/single-byte-decoder.window.js b/test/fixtures/wpt/encoding/single-byte-decoder.window.js
diff --git a/test/fixtures/wpt/encoding/streams/decode-utf8.any.js b/test/fixtures/wpt/encoding/streams/decode-utf8.any.js
@@ -43,7 +43,16 @@
     const array = await readableStreamToArray(output);
     assert_array_equals(array, [expectedOutputString],
                         'the output should be in one chunk');
-  }, 'a trailing empty chunk should be ignored- ' + arrayBufferOrSharedArrayBuffer);
+  }, 'a trailing empty chunk should be ignored - ' + arrayBufferOrSharedArrayBuffer);
+
+  promise_test(async () => {
+    const chunk = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, 3));
+    chunk.set([0xF0, 0x9F, 0x92]);
+    const input = readableStreamFromArray([chunk]);
+    const output = input.pipeThrough(new TextDecoderStream());
+    const array = await readableStreamToArray(output);
+    assert_array_equals(array, ['\uFFFD']);
+  }, 'UTF-8 EOF handling - ' + arrayBufferOrSharedArrayBuffer);
 });
 
 promise_test(async () => {

diff --git a/test/fixtures/wpt/encoding/streams/decode-utf8.any.js.headers b/test/fixtures/wpt/encoding/streams/decode-utf8.any.js.headers
@@ -0,0 +1,2 @@
+Cross-Origin-Opener-Policy: same-origin
+Cross-Origin-Embedder-Policy: require-corp
diff --git a/test/fixtures/wpt/encoding/textdecoder-copy.any.js.headers b/test/fixtures/wpt/encoding/textdecoder-copy.any.js.headers
@@ -0,0 +1,2 @@
+Cross-Origin-Opener-Policy: same-origin
+Cross-Origin-Embedder-Policy: require-corp
diff --git a/test/fixtures/wpt/encoding/textdecoder-eof.any.js b/test/fixtures/wpt/encoding/textdecoder-eof.any.js
@@ -0,0 +1,40 @@
+test(() => {
+  // Truncated sequences
+  assert_equals(new TextDecoder().decode(new Uint8Array([0xF0])), "\uFFFD");
+  assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F])), "\uFFFD");
+  assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F, 0x92])), "\uFFFD");
+
+  // Errors near end-of-queue
+  assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F, 0x41])), "\uFFFDA");
+  assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x41, 0x42])), "\uFFFDAB");
+  assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x41, 0xF0])), "\uFFFDA\uFFFD");
+  assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x8F, 0x92])), "\uFFFD\uFFFD\uFFFD");
+}, "TextDecoder end-of-queue handling");
+
+test(() => {
+  const decoder = new TextDecoder();
+  decoder.decode(new Uint8Array([0xF0]), { stream: true });
+  assert_equals(decoder.decode(), "\uFFFD");
+
+  decoder.decode(new Uint8Array([0xF0]), { stream: true });
+  decoder.decode(new Uint8Array([0x9F]), { stream: true });
+  assert_equals(decoder.decode(), "\uFFFD");
+
+  decoder.decode(new Uint8Array([0xF0, 0x9F]), { stream: true });
+  assert_equals(decoder.decode(new Uint8Array([0x92])), "\uFFFD");
+
+  assert_equals(decoder.decode(new Uint8Array([0xF0, 0x9F]), { stream: true }), "");
+  assert_equals(decoder.decode(new Uint8Array([0x41]), { stream: true }), "\uFFFDA");
+  assert_equals(decoder.decode(), "");
+
+  assert_equals(decoder.decode(new Uint8Array([0xF0, 0x41, 0x42]), { stream: true }), "\uFFFDAB");
+  assert_equals(decoder.decode(), "");
+
+  assert_equals(decoder.decode(new Uint8Array([0xF0, 0x41, 0xF0]), { stream: true }), "\uFFFDA");
+  assert_equals(decoder.decode(), "\uFFFD");
+
+  assert_equals(decoder.decode(new Uint8Array([0xF0]), { stream: true }), "");
+  assert_equals(decoder.decode(new Uint8Array([0x8F]), { stream: true }), "\uFFFD\uFFFD");
+  assert_equals(decoder.decode(new Uint8Array([0x92]), { stream: true }), "\uFFFD");
+  assert_equals(decoder.decode(), "");
+}, "TextDecoder end-of-queue handling using stream: true");
diff --git a/test/fixtures/wpt/encoding/textdecoder-streaming.any.js b/test/fixtures/wpt/encoding/textdecoder-streaming.any.js
@@ -28,15 +28,62 @@ var octets = {
                 var decoder = new TextDecoder(encoding);
                 for (var i = 0; i < encoded.length; i += len) {
                     var sub = [];
-                    for (var j = i; j < encoded.length && j < i + len; ++j)
+                    for (var j = i; j < encoded.length && j < i + len; ++j) {
                         sub.push(encoded[j]);
-                        var uintArray = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, sub.length));
-                        uintArray.set(sub);
+                    }
+                    var uintArray = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, sub.length));
+                    uintArray.set(sub);
                     out += decoder.decode(uintArray, {stream: true});
                 }
                 out += decoder.decode();
                 assert_equals(out, string);
             }, 'Streaming decode: ' + encoding + ', ' + len + ' byte window (' + arrayBufferOrSharedArrayBuffer + ')');
         }
     });
+
+    test(() => {
+        function bytes(byteArray) {
+            const view = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, byteArray.length));
+            view.set(byteArray);
+            return view;
+        }
+
+        const decoder = new TextDecoder();
+
+        assert_equals(decoder.decode(bytes([0xC1]), {stream: true}), "\uFFFD");
+        assert_equals(decoder.decode(), "");
+
+        assert_equals(decoder.decode(bytes([0xF5]), {stream: true}), "\uFFFD");
+        assert_equals(decoder.decode(), "");
+
+        assert_equals(decoder.decode(bytes([0xE0, 0x41]), {stream: true}), "\uFFFDA");
+        assert_equals(decoder.decode(bytes([0x42])), "B");
+
+        assert_equals(decoder.decode(bytes([0xE0, 0x80]), {stream: true}), "\uFFFD\uFFFD");
+        assert_equals(decoder.decode(bytes([0x80])), "\uFFFD");
+
+        assert_equals(decoder.decode(bytes([0xED, 0xA0]), {stream: true}), "\uFFFD\uFFFD");
+        assert_equals(decoder.decode(bytes([0x80])), "\uFFFD");
+
+        assert_equals(decoder.decode(bytes([0xF0, 0x41]), {stream: true}), "\uFFFDA");
+        assert_equals(decoder.decode(bytes([0x42]), {stream: true}), "B");
+        assert_equals(decoder.decode(bytes([0x43])), "C");
+
+        assert_equals(decoder.decode(bytes([0xF0, 0x80]), {stream: true}), "\uFFFD\uFFFD");
+        assert_equals(decoder.decode(bytes([0x80]), {stream: true}), "\uFFFD");
+        assert_equals(decoder.decode(bytes([0x80])), "\uFFFD");
+
+        assert_equals(decoder.decode(bytes([0xF4, 0xA0]), {stream: true}), "\uFFFD\uFFFD");
+        assert_equals(decoder.decode(bytes([0x80]), {stream: true}), "\uFFFD");
+        assert_equals(decoder.decode(bytes([0x80])), "\uFFFD");
+
+        assert_equals(decoder.decode(bytes([0xF0, 0x90, 0x41]), {stream: true}), "\uFFFDA");
+        assert_equals(decoder.decode(bytes([0x42])), "B");
+
+        // 4-byte UTF-8 sequences always correspond to non-BMP characters. Here
+        // we make sure that, although the first 3 bytes are enough to emit the
+        // lead surrogate, it only gets emitted when the fourth byte is read.
+        assert_equals(decoder.decode(bytes([0xF0, 0x9F, 0x92]), {stream: true}), "");
+        assert_equals(decoder.decode(bytes([0xA9])), "\u{1F4A9}");
+    }, `Streaming decode: UTF-8 chunk tests (${arrayBufferOrSharedArrayBuffer})`);
 })
diff --git a/test/fixtures/wpt/encoding/textdecoder-streaming.any.js.headers b/test/fixtures/wpt/encoding/textdecoder-streaming.any.js.headers
@@ -0,0 +1,2 @@
+Cross-Origin-Opener-Policy: same-origin
+Cross-Origin-Embedder-Policy: require-corp
diff --git a/test/fixtures/wpt/versions.json b/test/fixtures/wpt/versions.json
@@ -16,7 +16,7 @@
     "path": "dom/events"
   },
   "encoding": {
-    "commit": "35f70910d3753c8b650fdfd4c716caedfefe88c9",
+    "commit": "c1b24fce6e625c1b79124a58f27bf9adce02d5d7",
     "path": "encoding"
   },
   "FileAPI": {

diff --git a/test/wpt/status/encoding.json b/test/wpt/status/encoding.json
@@ -15,8 +15,7 @@
     "requires": ["small-icu"]
   },
   "textdecoder-streaming.any.js": {
-    "requires": ["small-icu"],
-    "fail": "WebAssembly.Memory does not support shared:true"
+    "requires": ["small-icu"]
   },
   "textdecoder-utf16-surrogates.any.js": {
     "requires": ["small-icu"]
@@ -52,12 +51,6 @@
   "streams/*.js": {
     "fail": "No implementation of TextDecoderStream and TextEncoderStream"
   },
-  "encodeInto.any.js": {
-    "fail": "WebAssembly.Memory does not support shared:true"
-  },
-  "textdecoder-copy.any.js": {
-    "fail": "WebAssembly.Memory does not support shared:true"
-  },
   "legacy-mb-schinese/gbk/gbk-decoder.any.js": {
     "requires": ["full-icu"],
     "skip": "The gbk encoding is not supported"
@@ -66,7 +59,7 @@
     "requires": ["full-icu"],
     "skip": "The gb18030 encoding is not supported"
   },
-  "textdecoder-arguments.any.js": {
-    "fail": "Does not support flushing an incomplete sequence"
+  "single-byte-decoder.window.js": {
+    "skip": "location is not defined"
   }
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Cross-Origin-Opener-Policy: same-origin
		Cross-Origin-Embedder-Policy: require-corp
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		// META: script=/resources/idlharness-shadowrealm.js
		idl_test_shadowrealm(["encoding"], ["streams"]);
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Cross-Origin-Opener-Policy:same-origin
		Cross-Origin-Embedder-Policy:require-corp