feat(source-memory): add memory source

packages/__tests__/index.js → packages/__tests__/index.test.js

@@ -1,15 +1,17 @@
 import S3v3 from '@aws-sdk/client-s3';
+import { fsa } from '@chunkd/fs';
 import { FsAwsS3 } from '@chunkd/source-aws';
 import { S3LikeV3 } from '@chunkd/source-aws-v3';
 import { FsGoogleStorage } from '@chunkd/source-google-cloud';
+import { FsMemory } from '@chunkd/source-memory';
 import { Storage } from '@google-cloud/storage';
 import S3 from 'aws-sdk/clients/s3.js';
 import o from 'ospec';
-import { fsa } from '../fs/build/index.node.js';
 
 fsa.register(`s3://blacha-chunkd-test/v2`, new FsAwsS3(new S3()));
 fsa.register(`s3://blacha-chunkd-test/v3`, new FsAwsS3(new S3LikeV3(new S3v3.S3())));
 fsa.register(`gs://blacha-chunkd-test/`, new FsGoogleStorage(new Storage()));
+fsa.register(`memory://blacha-chunkd-test/`, new FsMemory());
 
 const TestFiles = [
  { path: 'a/b/file-a-b-1.txt', buffer: Buffer.from('a/b/file-a-b-1.txt') },
@@ -99,9 +101,14 @@ function testPrefix(prefix) {
  });
 }
 
-testPrefix('/tmp/blacha-chunkd-test/');
+// testPrefix('/tmp/blacha-chunkd-test/');
+testPrefix('memory://blacha-chunkd-test/');
 // testPrefix('s3://blacha-chunkd-test/v2/');
 // testPrefix('s3://blacha-chunkd-test/v3/');
 // testPrefix('gs://blacha-chunkd-test/');
 
-o.run();
+// o.run();
+
+// run it directly when not included by ospec
+if (process.argv.find((f) => f.includes('.bin/ospec') == null)) o.run();
+// console.log(process.argv);

packages/core/src/index.ts

@@ -33,3 +33,23 @@ export function parseUri(uri: string): { protocol: string; bucket: string; key?:
  if (key == null || key.trim() === '') return { protocol, bucket };
  return { key, bucket, protocol };
 }
+
+const endsWithSlash = /\/$/;
+const startsWithSlash = /^\//;
+/** path.join removes slashes, s3:// => s3:/ which causes issues */
+export function joinUri(filePathA: string, filePathB: string): string {
+ return filePathA.replace(endsWithSlash, '') + '/' + filePathB.replace(startsWithSlash, '');
+}
+
+export function joinAllUri(filePathA: string, ...filePaths: string[]): string {
+ let output = filePathA;
+ for (let i = 0; i < filePaths.length; i++) output = joinUri(output, filePaths[i]);
+ return output;
+}
+
+/** Utility to convert async generators into arrays */
+export async function toArray<T>(generator: AsyncGenerator<T>): Promise<T[]> {
+ const output: T[] = [];
+ for await (const o of generator) output.push(o);
+ return output;
+}

packages/fs/src/fs.abstraction.ts

@@ -1,6 +1,14 @@
+import {
+ ChunkSource,
+ FileInfo,
+ FileSystem,
+ joinAllUri,
+ joinUri,
+ ListOptions,
+ toArray,
+ WriteOptions,
+} from '@chunkd/core';
 import type { Readable } from 'stream';
-import { ChunkSource, FileInfo, FileSystem, WriteOptions } from '@chunkd/core';
-import { ListOptions } from '@chunkd/core';
 
 export type FileWriteTypes = Buffer | Readable | string | Record<string, unknown> | Array<unknown>;
 
@@ -40,13 +48,7 @@ export class FileSystemAbstraction implements FileSystem {
  this.isOrdered = false;
  }
 
- /** Utility to convert async generators into arrays */
- async toArray<T>(generator: AsyncGenerator<T>): Promise<T[]> {
- const output: T[] = [];
- for await (const o of generator) output.push(o);
- return output;
- }
-
+ toArray = toArray;
  /**
  * Read a file into memory
  *
@@ -134,10 +136,8 @@ export class FileSystemAbstraction implements FileSystem {
  return this.get(filePath).head(filePath);
  }
 
- /** path.join removes slashes, s3:// => s3:/ which causes issues */
- join(filePathA: string, filePathB: string): string {
- return filePathA.replace(/\/$/, '') + '/' + filePathB.replace(/^\//, '');
- }
+ join = joinUri;
+ joinAll = joinAllUri;
 
  /**
  * create a chunked reading source from the file path

packages/source-memory/.npmignore

@@ -0,0 +1 @@
+tsconfig.tsbuildinfo

packages/source-memory/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+

packages/source-memory/README.md

@@ -0,0 +1,18 @@
+# @chunkd/source-memory
+
+Use memory as a simple file system, 
+
+this is designed for unit tests to prevent file system access, and not recommended for large file workloads.
+
+## Usage
+
+```javascript
+import { FsMemory } from '@chunkd/source-memory';
+
+fsa.register('memory://', new FsMemory());
+
+await fsa.write('memory://foo.png', pngBuffer);
+
+await fsa.read('memory://foo.png'); // png
+
+```

packages/source-memory/package.json

@@ -0,0 +1,27 @@
+{
+ "name": "@chunkd/source-memory",
+ "version": "8.2.0",
+ "type": "module",
+ "engines": {
+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+ },
+ "repository": {
+ "type": "git",
+ "url": "https://github.com/blacha/chunkd.git",
+ "directory": "packages/source-memory"
+ },
+ "main": "./build/index.js",
+ "types": "./build/index.d.ts",
+ "author": "Blayne Chard",
+ "license": "MIT",
+ "scripts": {},
+ "dependencies": {
+ "@chunkd/core": "^8.2.0"
+ },
+ "devDependencies": {
+ "@types/node": "^17.0.35"
+ },
+ "publishConfig": {
+ "access": "public"
+ }
+}

packages/source-memory/src/__test__/source.memory.test.ts

@@ -0,0 +1,36 @@
+import { toArray } from '@chunkd/core';
+import o from 'ospec';
+import { FsMemory } from '../memory.fs.js';
+
+o.spec('FsMemory', () => {
+ const memory = new FsMemory();
+
+ o.afterEach(() => {
+ memory.files.clear();
+ });
+ o('should write files', async () => {
+ o(memory.files.size).equals(0);
+ await memory.write('memory://foo.png', Buffer.from('a'));
+ o(memory.files.size).equals(1);
+
+ const b = await memory.read('memory://foo.png');
+ o(b.toString()).equals('a');
+ });
+
+ o('should stream files', async () => {
+ await memory.write('memory://foo.png', Buffer.from('a'));
+
+ await memory.write('memory://bar.png', memory.stream('memory://foo.png'));
+ o(memory.files.size).equals(2);
+
+ const bar = await memory.read('memory://bar.png');
+ o(bar.toString()).equals('a');
+ });
+
+ o('should list files', async () => {
+ await memory.write('memory://a/b/c.png', Buffer.from('a'));
+ await memory.write('memory://a/d.png', Buffer.from('a'));
+
+ o(await toArray(memory.list('memory://a/b'))).deepEquals(['memory://a/b/c.png']);
+ });
+});

packages/source-memory/src/index.ts

@@ -0,0 +1,2 @@
+export { SourceMemory } from '@chunkd/core';
+export { FsMemory } from './memory.fs.js';

packages/source-memory/src/memory.fs.ts

@@ -0,0 +1,98 @@
+import { CompositeError, FileInfo, FileSystem, ListOptions, SourceMemory } from '@chunkd/core';
+import { Readable } from 'stream';
+
+export function toReadable(r: string | Buffer | Readable): Readable {
+ if (typeof r === 'string') r = Buffer.from(r);
+ return Readable.from(r);
+}
+
+export async function toBuffer(stream: Readable): Promise<Buffer> {
+ return new Promise<Buffer>((resolve, reject) => {
+ const buf: Buffer[] = [];
+
+ stream.on('data', (chunk) => buf.push(chunk));
+ stream.on('end', () => resolve(Buffer.concat(buf)));
+ stream.on('error', (err) => reject(`error converting stream - ${err}`));
+ });
+}
+
+export class FsMemory implements FileSystem<SourceMemory> {
+ protocol = 'memory';
+
+ files: Map<string, Buffer> = new Map();
+
+ async read(filePath: string): Promise<Buffer> {
+ const data = this.files.get(filePath);
+ if (data == null) throw new CompositeError('Not found', 404, new Error());
+ return data;
+ }
+
+ stream(filePath: string): Readable {
+ const buf = this.files.get(filePath);
+ if (buf == null) throw new CompositeError('Not found', 404, new Error());
+ return toReadable(buf);
+ }
+
+ async write(filePath: string, buffer: string | Buffer | Readable): Promise<void> {
+ if (typeof buffer === 'string') {
+ this.files.set(filePath, Buffer.from(buffer));
+ return;
+ }
+ if (Buffer.isBuffer(buffer)) {
+ this.files.set(filePath, buffer);
+ return;
+ }
+ const buf = await toBuffer(buffer);
+ this.files.set(filePath, buf);
+ }
+
+ async *list(filePath: string, opt?: ListOptions): AsyncGenerator<string> {
+ const folders = new Set();
+ for (const file of this.files.keys()) {
+ if (file.startsWith(filePath)) {
+ if (opt?.recursive === false) {
+ const subPath = file.slice(filePath.length);
+ const parts = subPath.split('/');
+ if (parts.length === 1) yield file;
+ else {
+ const folderName = parts[0];
+ if (folders.has(folderName)) continue;
+ folders.add(folderName);
+ yield filePath + folderName + '/';
+ }
+ } else {
+ yield file;
+ }
+ }
+ }
+ }
+
+ async *details(filePath: string, opt?: ListOptions): AsyncGenerator<FileInfo> {
+ for await (const file of this.list(filePath, opt)) {
+ const data = await this.head(file);
+ if (data == null) {
+ yield { path: file, isDirectory: true };
+ } else {
+ yield data;
+ }
+ }
+ }
+
+ async exists(filePath: string): Promise<boolean> {
+ const dat = await this.head(filePath);
+ return dat != null;
+ }
+
+ async head(filePath: string): Promise<FileInfo | null> {
+ const buf = this.files.get(filePath);
+ if (buf == null) return null;
+ return { path: filePath, size: buf.length };
+ }
+
+ source(filePath: string): SourceMemory {
+ const bytes = this.files.get(filePath);
+ if (bytes == null) throw new CompositeError('File not found', 404, new Error());
+ const source = new SourceMemory(filePath, bytes);
+ return source;
+ }
+}

packages/source-memory/tsconfig.json

@@ -0,0 +1,10 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "rootDir": "./src",
+ "outDir": "./build",
+ "lib": ["es2018", "DOM"]
+ },
+ "include": ["src/**/*", "src/.ts"],
+ "references": [{ "path": "../core" }]
+}

tsconfig.json

@@ -8,6 +8,7 @@
  { "path": "./packages/source-aws-v3" },
  { "path": "./packages/source-google-cloud" },
  { "path": "./packages/source-file" },
+ { "path": "./packages/source-memory" },
  { "path": "./packages/fs" }
  ]
 }