Version counters

[keller-mark]

I'm not sure if this is the best way to implement this (and especially the typings), but the general idea is that the root of the store can (optionally) maintain two counters: v2_count and v3_count, which represent the number of times the store has successfully opened v2 vs. v3 metadata. Then, we can use these counters to determine which version to prioritize in the subsequent open() calls.

The goal is to avoid extra v3 zarr.json network requests and console errors when requesting v2 data, but without restricting by using open.v2 or open.v3. This is particularly important in higlass-zarr-datafetchers https://github.com/higlass/higlass-zarr-datafetchers/tree/keller-mark-patch-1 where many requests are being made on zoom/pan interactions.

[manzt]

I really like the idea, but in terms of implementation I'm not sure it's something that should live on the store. I think this might be a nice use case for a WeakMap, where we could keep context so long as the store lives in memory.

// open.ts
function create_version_counter() {
  let version_counts = new WeakMap<Readable, { v2: number, v3: number}>();
  function get_counts(store: Readable) {
    let counts = version_counts.get(store) ?? { v2: 0, v3: 0 };
    version_counts.set(store, counts);
    return counts;
  }
  return {
    increment(store: Readable, version: "v2" | "v3") {
      get_counts(store)[version] += 1;
    },
    version_max(store: Readable): "v2" | "v3" {
      let counts = get_counts(store);
      return counts.v3 > counts.v2 ? "v3" :  "v2";
    }
  }
}
let VERSION_COUNTER = create_version_counter();

[keller-mark]

Ah that is much nicer! I have re-implemented using your code snippet

[manzt]

Looks great. Just a comment on when we should increment the counts.

[manzt]

For v2, maybe we only increment if a node has successfully been found? this within open_array_v2 and open_group_v2 but only if they are successful?

async function open_array_v2<Store extends Readable>(
	location: Location<Store>,
	attrs: Attributes,
) {
	let { path } = location.resolve(".zarray");
	let meta = await location.store.get(path);
	if (!meta) {
		throw new NodeNotFoundError(path);
	}
++	VERSION_COUNTER.increment(loc.store, "v2");
	return new Array(
		location.store,
		location.path,
		v2_to_v3_array_metadata(json_decode_object(meta), attrs),
	);
}

Or we add some wrappers functions:

async function open_v2_with_counter<Store extends Readable>(...args: Parameters<typeof open_v2<Store>>) {
	let node = await open_v2(...args);
	VERSION_COUNTER.increment(loc.store, "v2");
	return node;
}