Use UTF-8 collation in lexicographicCompare, closes #380

cloudflare · Sep 29, 2022 · d6eff77 · d6eff77
1 parent 9096997
commit d6eff77
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 4 deletions.
diff --git a/packages/durable-objects/test/storage.spec.ts b/packages/durable-objects/test/storage.spec.ts
@@ -876,8 +876,14 @@ test("list: sorts lexicographically", async (t) => {
   // https://github.com/cloudflare/miniflare/issues/235
   const { storage } = t.context;
   await storage.put({ "!": {}, ", ": {} });
-  const keys = Array.from((await storage.list()).keys());
+  let keys = Array.from((await storage.list()).keys());
   t.deepEqual(keys, ["!", ", "]);
+
+  // https://github.com/cloudflare/miniflare/issues/380
+  await storage.deleteAll();
+  await storage.put({ Z: 0, "\u{1D655}": 1, "\uFF3A": 2 });
+  keys = Array.from((await storage.list()).keys());
+  t.deepEqual(keys, ["Z", "\uFF3A", "\u{1D655}"]);
 });
 test("list: closes input gate unless allowConcurrency", async (t) => {
   const { storage } = t.context;

diff --git a/packages/shared/src/data.ts b/packages/shared/src/data.ts
@@ -1,14 +1,32 @@
 import path from "path";
+import { TextEncoder } from "util";
 import { deserialize, serialize } from "v8";
 import picomatch from "picomatch";
 
+const encoder = new TextEncoder();
+
 export const numericCompare = new Intl.Collator(undefined, { numeric: true })
   .compare;
 
+export function arrayCompare<T extends any[] | NodeJS.TypedArray>(
+  a: T,
+  b: T
+): number {
+  const minLength = Math.min(a.length, b.length);
+  for (let i = 0; i < minLength; i++) {
+    const aElement = a[i];
+    const bElement = b[i];
+    if (aElement < bElement) return -1;
+    if (aElement > bElement) return 1;
+  }
+  return a.length - b.length;
+}
+
+// Compares x and y lexicographically using a UTF-8 collation
 export function lexicographicCompare(x: string, y: string): number {
-  if (x < y) return -1;
-  if (x === y) return 0;
-  return 1;
+  const xEncoded = encoder.encode(x);
+  const yEncoded = encoder.encode(y);
+  return arrayCompare(xEncoded, yEncoded);
 }
 
 export function nonCircularClone<T>(value: T): T {

diff --git a/packages/shared/test/data.spec.ts b/packages/shared/test/data.spec.ts
@@ -2,6 +2,7 @@ import path from "path";
 import { TextDecoder } from "util";
 import {
   addAll,
+  arrayCompare,
   base64Decode,
   base64Encode,
   globsToMatcher,
@@ -18,11 +19,32 @@ import {
 import { useTmp } from "@miniflare/shared-test";
 import test from "ava";
 
+test("arrayCompare: compares arrays", (t) => {
+  // Check with numeric values
+  t.is(arrayCompare([], []), 0);
+  t.is(arrayCompare([1, 2, 3], [1, 2, 3]), 0);
+  t.true(arrayCompare([], [1]) < 0);
+  t.true(arrayCompare([1], []) > 0);
+  t.true(arrayCompare([1, 2, 3], [1, 2, 4]) < 0);
+  t.true(arrayCompare([1, 2, 4], [1, 2, 3]) > 0);
+  t.true(arrayCompare([1, 2], [1, 2, 3]) < 0);
+  t.true(arrayCompare([1, 2, 3], [1, 2]) > 0);
+
+  // Check with non-numeric values
+  t.true(arrayCompare(["a", "b", "c"], ["a", "b", "d"]) < 0);
+  t.true(arrayCompare(["a", "b", "d"], ["a", "b", "c"]) > 0);
+});
+
 test("lexicographicCompare: compares lexicographically", (t) => {
   t.is(lexicographicCompare("a", "b"), -1);
   t.is(lexicographicCompare("a", "a"), 0);
   t.is(lexicographicCompare("b", "a"), 1);
   t.is(lexicographicCompare("!", ", "), -1);
+
+  // https://github.com/cloudflare/miniflare/issues/380
+  t.is(lexicographicCompare("Z", "\uFF3A"), -1);
+  t.is(lexicographicCompare("\uFF3A", "\u{1D655}"), -1);
+  t.is(lexicographicCompare("\u{1D655}", "Z"), 1);
 });
 
 test("nonCircularClone: creates copy of data", (t) => {