feat(fsa): decompress "json.gz" files automatically with readJson (#1345

)

packages/fs/src/__test__/fs.abstraction.test.ts

@@ -2,6 +2,8 @@ import { FsFile } from '../systems/file.js';
 import { describe, it } from 'node:test';
 import assert from 'node:assert';
 import { FileSystemAbstraction } from '../file.system.abstraction.js';
+import { gzipSync } from 'node:zlib';
+import { FsMemory } from '../systems/memory.js';
 
 export class FakeSystem extends FsFile {
  constructor(protocol = 'fake') {
@@ -111,4 +113,39 @@ describe('FileSystemAbstraction', () => {
  assert.equal(fsa.toUrl('s3://foo/bar').href, 's3://foo/bar');
  assert.equal(fsa.toUrl('fake-a://fake.com/bar').href, 'fake-a://fake.com/bar');
  });
+
+ describe('readJson', () => {
+ it('should readJson objects', async () => {
+ const fsa = new FileSystemAbstraction();
+ const fsMem = new FsMemory();
+ fsa.register('memory://', fsMem);
+
+ await fsa.write(new URL('memory://memory/empty.json'), Buffer.from(JSON.stringify({ Hello: 'World' })));
+ const result = await fsa.readJson(new URL('memory://memory/empty.json'));
+ assert.deepEqual(result, { Hello: 'World' });
+ });
+
+ it('should readJson gzipped objects', async () => {
+ const fsa = new FileSystemAbstraction();
+ const fsMem = new FsMemory();
+ fsa.register('memory://', fsMem);
+
+ await fsa.write(new URL('memory://memory/empty.json'), gzipSync(Buffer.from(JSON.stringify({ Hello: 'World' }))));
+ const result = await fsa.readJson(new URL('memory://memory/empty.json'));
+ assert.deepEqual(result, { Hello: 'World' });
+ });
+
+ it('should readJson gzipped objects ending in ".gz"', async () => {
+ const fsa = new FileSystemAbstraction();
+ const fsMem = new FsMemory();
+ fsa.register('memory://', fsMem);
+
+ await fsa.write(
+ new URL('memory://memory/empty.json.gz'),
+ gzipSync(Buffer.from(JSON.stringify({ Hello: 'World' }))),
+ );
+ const result = await fsa.readJson(new URL('memory://memory/empty.json.gz'));
+ assert.deepEqual(result, { Hello: 'World' });
+ });
+ });
 });

packages/fs/src/file.system.abstraction.ts

@@ -1,11 +1,27 @@
 import { Source, SourceMiddleware, SourceView } from '@chunkd/source';
 import type { Readable } from 'node:stream';
 import { pathToFileURL } from 'node:url';
+import { gunzip } from 'node:zlib';
+import { promisify } from 'node:util';
 import { FileInfo, FileSystem, FileWriteTypes, ListOptions, WriteOptions } from './file.system.js';
 import { Flag } from './flags.js';
 import { FsFile } from './systems/file.js';
 import { toArray } from './generator.js';
 
+const gunzipProm = promisify(gunzip);
+
+/**
+ * Does a buffer look like a gzipped document instead of raw json
+ *
+ * Determined by checking the first two bytes are the gzip magic bytes `0x1f 0x8b`
+ *
+ * @see https://en.wikipedia.org/wiki/Gzip
+ *
+ */
+export function isGzip(b: Buffer): boolean {
+ return b[0] === 0x1f && b[1] === 0x8b;
+}
+
 export class FileSystemAbstraction implements FileSystem {
  name = 'fsa';
  /**
@@ -67,11 +83,18 @@ export class FileSystemAbstraction implements FileSystem {
 
  /**
  * Read a file as JSON
+ *
+ * If the file ends with .gz or is a GZIP like {@link isGzip} file it will automatically be decompressed.
+ *
  * @param loc file to read
  * @returns JSON Content of the file
  */
  async readJson<T>(loc: URL): Promise<T> {
  const obj = await this.read(loc);
+ if (loc.pathname.endsWith('.gz') || isGzip(obj)) {
+ const data = await gunzipProm(obj);
+ return JSON.parse(data.toString());
+ }
  return JSON.parse(obj.toString());
  }