From 191d95c77d2c7902344cd0175ae0044f740d19ba Mon Sep 17 00:00:00 2001 From: Trevor Manz Date: Sun, 14 Jan 2024 14:23:41 -0500 Subject: [PATCH] feat(core): Add `tryWithConsolidated` helper (#141) --- .changeset/green-pandas-argue.md | 23 ++++++++ packages/core/__tests__/consolidated.test.ts | 32 ++++++++++- packages/core/__tests__/open.test.ts | 18 ++++-- packages/core/src/consolidated.ts | 58 ++++++++++++++++---- packages/core/src/index.ts | 1 + packages/zarrita/index.test.ts | 2 + 6 files changed, 116 insertions(+), 18 deletions(-) create mode 100644 .changeset/green-pandas-argue.md diff --git a/.changeset/green-pandas-argue.md b/.changeset/green-pandas-argue.md new file mode 100644 index 00000000..b72968d0 --- /dev/null +++ b/.changeset/green-pandas-argue.md @@ -0,0 +1,23 @@ +--- +"@zarrita/core": minor +--- + +feat: Add `tryWithConsolidated` store helper + +Provides a convenient way to open a store that may or may not have consolidated +metadata. Ideal for usage senarios with known access paths, since store with +consolidated metadata do not incur additional network requests when accessing +underlying groups and arrays. + +```js +import * as zarr from "zarrita"; + +let store = await zarr.tryWithConsolidated( + new zarr.FetchStore("https://localhost:8080/data.zarr"); +); + +// The following do not read from the store +// (make network requests) if it is consolidated. +let grp = await zarr.open(store, { kind: "group" }); +let foo = await zarr.open(grp.resolve("foo"), { kind: "array" }); +``` diff --git a/packages/core/__tests__/consolidated.test.ts b/packages/core/__tests__/consolidated.test.ts index e3831333..9bfc021d 100644 --- a/packages/core/__tests__/consolidated.test.ts +++ b/packages/core/__tests__/consolidated.test.ts @@ -3,9 +3,10 @@ import * as path from "node:path"; import * as url from "node:url"; import { FileSystemStore } from "@zarrita/storage"; -import { withConsolidated } from "../src/consolidated.js"; +import { tryWithConsolidated, withConsolidated } from "../src/consolidated.js"; import { open } from "../src/open.js"; import { Array as ZarrArray } from "../src/hierarchy.js"; +import { NodeNotFoundError } from "../src/errors.js"; let __dirname = path.dirname(url.fileURLToPath(import.meta.url)); @@ -94,4 +95,33 @@ describe("withConsolidated", () => { let arr = await open(grp.resolve("1d.chunked.i2"), { kind: "array" }); expect(arr.kind).toBe("array"); }); + + it("throws if consolidated metadata is missing", async () => { + let root = path.join( + __dirname, + "../../../fixtures/v2/data.zarr/3d.contiguous.i2", + ); + let try_open = () => withConsolidated(new FileSystemStore(root)); + await expect(try_open).rejects.toThrowError(NodeNotFoundError); + await expect(try_open).rejects.toThrowErrorMatchingInlineSnapshot( + '"Node not found: v2 consolidated metadata"', + ); + }); +}); + +describe("tryWithConsolidated", () => { + it("creates Listable from consolidated store", async () => { + let root = path.join(__dirname, "../../../fixtures/v2/data.zarr"); + let store = await tryWithConsolidated(new FileSystemStore(root)); + expect(store).toHaveProperty("contents"); + }); + + it("falls back to original store if missing consolidated metadata", async () => { + let root = path.join( + __dirname, + "../../../fixtures/v2/data.zarr/3d.contiguous.i2", + ); + let store = await tryWithConsolidated(new FileSystemStore(root)); + expect(store).toBeInstanceOf(FileSystemStore); + }); }); diff --git a/packages/core/__tests__/open.test.ts b/packages/core/__tests__/open.test.ts index 234a5ebe..c5906b61 100644 --- a/packages/core/__tests__/open.test.ts +++ b/packages/core/__tests__/open.test.ts @@ -463,9 +463,12 @@ describe("v2", () => { }); it("throws when group is not found", async () => { - await expect(open.v2(store.resolve("/not/a/group"), { kind: "group" })) - .rejects - .toThrow(NodeNotFoundError); + let try_open = () => + open.v2(store.resolve("/not/a/group"), { kind: "group" }); + await expect(try_open).rejects.toThrowError(NodeNotFoundError); + await expect(try_open).rejects.toThrowErrorMatchingInlineSnapshot( + '"Node not found: v2 group"', + ); }); describe("opens array from group", async () => { @@ -717,9 +720,12 @@ describe("v3", () => { }); it("throws when group not found", async () => { - await expect(open.v3(store.resolve("/not/a/group"), { kind: "group" })) - .rejects - .toThrow(NodeNotFoundError); + const try_open = () => + open.v3(store.resolve("/not/a/group"), { kind: "group" }); + await expect(try_open).rejects.toThrowError(NodeNotFoundError); + await expect(try_open).rejects.toThrowErrorMatchingInlineSnapshot( + '"Node not found: v3 array or group"', + ); }); describe("opens array from group", async () => { diff --git a/packages/core/src/consolidated.ts b/packages/core/src/consolidated.ts index a5f039e9..c49f07af 100644 --- a/packages/core/src/consolidated.ts +++ b/packages/core/src/consolidated.ts @@ -55,20 +55,33 @@ function is_v3(meta: Metadata): meta is ArrayMetadata | GroupMetadata { return "zarr_format" in meta && meta.zarr_format === 3; } +/** + * Open a consolidated store. + * + * This will open a store with Zarr v2 consolidated metadata (`.zmetadata`). + * @see {@link https://zarr.readthedocs.io/en/stable/spec/v2.html#consolidated-metadata} + * + * @param store The store to open. + * @returns A listable store. + * + * @example + * ```js + * let store = await withConsolidated( + * new zarr.FetchStore("https://my-bucket.s3.amazonaws.com"); + * ); + * store.contents(); // [{ path: "/", kind: "group" }, { path: "/foo", kind: "array" }, ...] + * let grp = zarr.open(store); // Open the root group. + * let foo = zarr.open(grp.resolve(contents[1].path)); // Open the foo array + * ``` + */ export async function withConsolidated( store: Store, ): Promise> { - let known_meta: Record = - await get_consolidated_metadata(store) - .then((meta) => { - let new_meta: Record = {}; - for (let [key, value] of Object.entries(meta.metadata)) { - new_meta[`/${key}`] = value; - } - return new_meta; - }) - .catch(() => ({})); - + let v2_meta = await get_consolidated_metadata(store); + let known_meta: Record = {}; + for (let [key, value] of Object.entries(v2_meta.metadata)) { + known_meta[`/${key}`] = value; + } return { async get( ...args: Parameters @@ -100,3 +113,26 @@ export async function withConsolidated( }, }; } + +/** + * Try to open a consolidated store, but fall back to the original store if the + * consolidated metadata is missing. + * + * Provides a convenient way to open a store that may or may not have consolidated, + * returning a consistent interface for both cases. Ideal for usage senarios with + * known access paths, since store with consolidated metadata do not incur + * additional network requests when accessing underlying groups and arrays. + * + * @param store The store to open. + * @returns A listable store. + */ +export async function tryWithConsolidated( + store: Store, +): Promise | Store> { + return withConsolidated(store).catch((e: unknown) => { + if (e instanceof NodeNotFoundError) { + return store; + } + throw e; + }); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f8d7e5d9..c95726fe 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,4 +9,5 @@ export { export { open } from "./open.js"; export { create } from "./create.js"; export { registry } from "./codecs.js"; +export { tryWithConsolidated, withConsolidated } from "./consolidated.js"; export type * from "./metadata.js"; diff --git a/packages/zarrita/index.test.ts b/packages/zarrita/index.test.ts index 677c3b6b..74025ead 100644 --- a/packages/zarrita/index.test.ts +++ b/packages/zarrita/index.test.ts @@ -29,6 +29,8 @@ it("exports all the things", () => { "root": [Function], "set": [Function], "slice": [Function], + "tryWithConsolidated": [Function], + "withConsolidated": [Function], } `); });