feat(source-memory): add memory source (#372)

* feat: add a memory source

* fix: support non recursive list

* refactor: sort the package references

* refactor: cleanup lint

packages/__tests__/index.js → packages/__tests__/index.test.js

@@ -1,15 +1,17 @@
 import S3v3 from '@aws-sdk/client-s3';
+import { fsa } from '@chunkd/fs';
 import { FsAwsS3 } from '@chunkd/source-aws';
 import { S3LikeV3 } from '@chunkd/source-aws-v3';
 import { FsGoogleStorage } from '@chunkd/source-google-cloud';
+import { FsMemory } from '@chunkd/source-memory';
 import { Storage } from '@google-cloud/storage';
 import S3 from 'aws-sdk/clients/s3.js';
 import o from 'ospec';
-import { fsa } from '../fs/build/index.node.js';
 
 fsa.register(`s3://blacha-chunkd-test/v2`, new FsAwsS3(new S3()));
 fsa.register(`s3://blacha-chunkd-test/v3`, new FsAwsS3(new S3LikeV3(new S3v3.S3())));
 fsa.register(`gs://blacha-chunkd-test/`, new FsGoogleStorage(new Storage()));
+fsa.register(`memory://blacha-chunkd-test/`, new FsMemory());
 
 const TestFiles = [
   { path: 'a/b/file-a-b-1.txt', buffer: Buffer.from('a/b/file-a-b-1.txt') },
@@ -99,9 +101,14 @@ function testPrefix(prefix) {
   });
 }
 
-testPrefix('/tmp/blacha-chunkd-test/');
+// testPrefix('/tmp/blacha-chunkd-test/');
+testPrefix('memory://blacha-chunkd-test/');
 // testPrefix('s3://blacha-chunkd-test/v2/');
 // testPrefix('s3://blacha-chunkd-test/v3/');
 // testPrefix('gs://blacha-chunkd-test/');
 
-o.run();
+// o.run();
+
+// run it directly when not included by ospec
+if (process.argv.find((f) => f.includes('.bin/ospec') == null)) o.run();
+// console.log(process.argv);

packages/core/src/index.ts

@@ -33,3 +33,23 @@ export function parseUri(uri: string): { protocol: string; bucket: string; key?:
   if (key == null || key.trim() === '') return { protocol, bucket };
   return { key, bucket, protocol };
 }
+
+const endsWithSlash = /\/$/;
+const startsWithSlash = /^\//;
+/** path.join removes slashes, s3:// => s3:/ which causes issues */
+export function joinUri(filePathA: string, filePathB: string): string {
+  return filePathA.replace(endsWithSlash, '') + '/' + filePathB.replace(startsWithSlash, '');
+}
+
+export function joinAllUri(filePathA: string, ...filePaths: string[]): string {
+  let output = filePathA;
+  for (let i = 0; i < filePaths.length; i++) output = joinUri(output, filePaths[i]);
+  return output;
+}
+
+/** Utility to convert async generators into arrays */
+export async function toArray<T>(generator: AsyncGenerator<T>): Promise<T[]> {
+  const output: T[] = [];
+  for await (const o of generator) output.push(o);
+  return output;
+}

packages/fs/src/fs.abstraction.ts

@@ -1,6 +1,14 @@
+import {
+  ChunkSource,
+  FileInfo,
+  FileSystem,
+  joinAllUri,
+  joinUri,
+  ListOptions,
+  toArray,
+  WriteOptions,
+} from '@chunkd/core';
 import type { Readable } from 'stream';
-import { ChunkSource, FileInfo, FileSystem, WriteOptions } from '@chunkd/core';
-import { ListOptions } from '@chunkd/core';
 
 export type FileWriteTypes = Buffer | Readable | string | Record<string, unknown> | Array<unknown>;
 
@@ -40,13 +48,7 @@ export class FileSystemAbstraction implements FileSystem {
     this.isOrdered = false;
   }
 
-  /** Utility to convert async generators into arrays */
-  async toArray<T>(generator: AsyncGenerator<T>): Promise<T[]> {
-    const output: T[] = [];
-    for await (const o of generator) output.push(o);
-    return output;
-  }
-
+  toArray = toArray;
   /**
    * Read a file into memory
    *
@@ -134,10 +136,8 @@ export class FileSystemAbstraction implements FileSystem {
     return this.get(filePath).head(filePath);
   }
 
-  /** path.join removes slashes, s3:// => s3:/ which causes issues */
-  join(filePathA: string, filePathB: string): string {
-    return filePathA.replace(/\/$/, '') + '/' + filePathB.replace(/^\//, '');
-  }
+  join = joinUri;
+  joinAll = joinAllUri;
 
   /**
    * create a chunked reading source from the file path

packages/source-memory/.npmignore

@@ -0,0 +1 @@
+tsconfig.tsbuildinfo

packages/source-memory/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+

packages/source-memory/README.md

@@ -0,0 +1,18 @@
+# @chunkd/source-memory
+
+Use memory as a simple file system, 
+
+this is designed for unit tests to prevent file system access, and not recommended for large file workloads.
+
+## Usage
+
+```javascript
+import { FsMemory } from '@chunkd/source-memory';
+
+fsa.register('memory://', new FsMemory());
+
+await fsa.write('memory://foo.png', pngBuffer);
+
+await fsa.read('memory://foo.png'); // png
+
+```

packages/source-memory/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@chunkd/source-memory",
+  "version": "8.2.0",
+  "type": "module",
+  "engines": {
+    "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/blacha/chunkd.git",
+    "directory": "packages/source-memory"
+  },
+  "main": "./build/index.js",
+  "types": "./build/index.d.ts",
+  "author": "Blayne Chard",
+  "license": "MIT",
+  "scripts": {},
+  "dependencies": {
+    "@chunkd/core": "^8.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^17.0.35"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/source-memory/src/__test__/source.memory.test.ts

@@ -0,0 +1,36 @@
+import { toArray } from '@chunkd/core';
+import o from 'ospec';
+import { FsMemory } from '../memory.fs.js';
+
+o.spec('FsMemory', () => {
+  const memory = new FsMemory();
+
+  o.afterEach(() => {
+    memory.files.clear();
+  });
+  o('should write files', async () => {
+    o(memory.files.size).equals(0);
+    await memory.write('memory://foo.png', Buffer.from('a'));
+    o(memory.files.size).equals(1);
+
+    const b = await memory.read('memory://foo.png');
+    o(b.toString()).equals('a');
+  });
+
+  o('should stream files', async () => {
+    await memory.write('memory://foo.png', Buffer.from('a'));
+
+    await memory.write('memory://bar.png', memory.stream('memory://foo.png'));
+    o(memory.files.size).equals(2);
+
+    const bar = await memory.read('memory://bar.png');
+    o(bar.toString()).equals('a');
+  });
+
+  o('should list files', async () => {
+    await memory.write('memory://a/b/c.png', Buffer.from('a'));
+    await memory.write('memory://a/d.png', Buffer.from('a'));
+
+    o(await toArray(memory.list('memory://a/b'))).deepEquals(['memory://a/b/c.png']);
+  });
+});

packages/source-memory/src/index.ts

@@ -0,0 +1,2 @@
+export { SourceMemory } from '@chunkd/core';
+export { FsMemory } from './memory.fs.js';

packages/source-memory/src/memory.fs.ts

@@ -0,0 +1,98 @@
+import { CompositeError, FileInfo, FileSystem, ListOptions, SourceMemory } from '@chunkd/core';
+import { Readable } from 'stream';
+
+export function toReadable(r: string | Buffer | Readable): Readable {
+  if (typeof r === 'string') r = Buffer.from(r);
+  return Readable.from(r);
+}
+
+export async function toBuffer(stream: Readable): Promise<Buffer> {
+  return new Promise<Buffer>((resolve, reject) => {
+    const buf: Buffer[] = [];
+
+    stream.on('data', (chunk) => buf.push(chunk));
+    stream.on('end', () => resolve(Buffer.concat(buf)));
+    stream.on('error', (err) => reject(`error converting stream - ${err}`));
+  });
+}
+
+export class FsMemory implements FileSystem<SourceMemory> {
+  protocol = 'memory';
+
+  files: Map<string, Buffer> = new Map();
+
+  async read(filePath: string): Promise<Buffer> {
+    const data = this.files.get(filePath);
+    if (data == null) throw new CompositeError('Not found', 404, new Error());
+    return data;
+  }
+
+  stream(filePath: string): Readable {
+    const buf = this.files.get(filePath);
+    if (buf == null) throw new CompositeError('Not found', 404, new Error());
+    return toReadable(buf);
+  }
+
+  async write(filePath: string, buffer: string | Buffer | Readable): Promise<void> {
+    if (typeof buffer === 'string') {
+      this.files.set(filePath, Buffer.from(buffer));
+      return;
+    }
+    if (Buffer.isBuffer(buffer)) {
+      this.files.set(filePath, buffer);
+      return;
+    }
+    const buf = await toBuffer(buffer);
+    this.files.set(filePath, buf);
+  }
+
+  async *list(filePath: string, opt?: ListOptions): AsyncGenerator<string> {
+    const folders = new Set();
+    for (const file of this.files.keys()) {
+      if (file.startsWith(filePath)) {
+        if (opt?.recursive === false) {
+          const subPath = file.slice(filePath.length);
+          const parts = subPath.split('/');
+          if (parts.length === 1) yield file;
+          else {
+            const folderName = parts[0];
+            if (folders.has(folderName)) continue;
+            folders.add(folderName);
+            yield filePath + folderName + '/';
+          }
+        } else {
+          yield file;
+        }
+      }
+    }
+  }
+
+  async *details(filePath: string, opt?: ListOptions): AsyncGenerator<FileInfo> {
+    for await (const file of this.list(filePath, opt)) {
+      const data = await this.head(file);
+      if (data == null) {
+        yield { path: file, isDirectory: true };
+      } else {
+        yield data;
+      }
+    }
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    const dat = await this.head(filePath);
+    return dat != null;
+  }
+
+  async head(filePath: string): Promise<FileInfo | null> {
+    const buf = this.files.get(filePath);
+    if (buf == null) return null;
+    return { path: filePath, size: buf.length };
+  }
+
+  source(filePath: string): SourceMemory {
+    const bytes = this.files.get(filePath);
+    if (bytes == null) throw new CompositeError('File not found', 404, new Error());
+    const source = new SourceMemory(filePath, bytes);
+    return source;
+  }
+}

packages/source-memory/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./build",
+    "lib": ["es2018", "DOM"]
+  },
+  "include": ["src/**/*", "src/.ts"],
+  "references": [{ "path": "../core" }]
+}

tsconfig.json

@@ -8,6 +8,7 @@
     { "path": "./packages/source-aws-v3" },
     { "path": "./packages/source-google-cloud" },
     { "path": "./packages/source-file" },
+    { "path": "./packages/source-memory" },
     { "path": "./packages/fs" }
   ]
 }