-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(fs): support non recursive file listing (#370)
limit file listing to be non recursive with {recursive: false}- Loading branch information
Showing
11 changed files
with
265 additions
and
61 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| import S3v3 from '@aws-sdk/client-s3'; | ||
| import { FsAwsS3 } from '@chunkd/source-aws'; | ||
| import { S3LikeV3 } from '@chunkd/source-aws-v3'; | ||
| import { FsGoogleStorage } from '@chunkd/source-google-cloud'; | ||
| import { Storage } from '@google-cloud/storage'; | ||
| import S3 from 'aws-sdk/clients/s3.js'; | ||
| import o from 'ospec'; | ||
| import { fsa } from '../fs/build/index.node.js'; | ||
|
|
||
| fsa.register(`s3://blacha-chunkd-test/v2`, new FsAwsS3(new S3())); | ||
| fsa.register(`s3://blacha-chunkd-test/v3`, new FsAwsS3(new S3LikeV3(new S3v3.S3()))); | ||
| fsa.register(`gs://blacha-chunkd-test/`, new FsGoogleStorage(new Storage())); | ||
|
|
||
| const TestFiles = [ | ||
| { path: 'a/b/file-a-b-1.txt', buffer: Buffer.from('a/b/file-a-b-1.txt') }, | ||
| { path: 'a/b/file-a-b-2', buffer: Buffer.from('a/b/file-a-b-2') }, | ||
| { path: 'a/file-a-1', buffer: Buffer.from('file-a-1') }, | ||
| { path: 'c/file-c-1', buffer: Buffer.from('file-c-1') }, | ||
| { path: 'd/file-d-1', buffer: Buffer.from('file-d-1') }, | ||
| { path: 'file-1', buffer: Buffer.from('file-1') }, | ||
| { path: 'file-2', buffer: Buffer.from('file-2') }, | ||
| { path: '🦄.json', buffer: Buffer.from('🦄') }, | ||
| ]; | ||
|
|
||
| async function setupTestData(prefix) { | ||
| try { | ||
| const existing = await fsa.toArray(fsa.list(prefix)); | ||
| if (existing.length === TestFiles.length) return; | ||
| } catch (e) { | ||
| //noop | ||
| } | ||
| for (const file of TestFiles) { | ||
| const target = fsa.join(prefix, file.path); | ||
| console.log(target); | ||
| await fsa.write(target, file.buffer); | ||
| } | ||
| } | ||
|
|
||
| function removeSlashes(f) { | ||
| if (f.startsWith('/')) f = f.slice(1); | ||
| if (f.endsWith('/')) f = f.slice(0, f.length - 1); | ||
| return f; | ||
| } | ||
|
|
||
| function testPrefix(prefix) { | ||
| o.spec(prefix, () => { | ||
| o.specTimeout(5000); | ||
| o.before(async () => { | ||
| await setupTestData(prefix); | ||
| }); | ||
|
|
||
| o('should list recursive:default ', async () => { | ||
| const files = await fsa.toArray(fsa.list(prefix)); | ||
| o(files.length).equals(TestFiles.length); | ||
| }); | ||
|
|
||
| o('should list recursive:true ', async () => { | ||
| const files = await fsa.toArray(fsa.list(prefix, { recursive: true })); | ||
| o(files.length).equals(TestFiles.length); | ||
|
|
||
| for (const file of TestFiles) { | ||
| o(files.find((f) => f.endsWith(file.path))).notEquals(undefined); | ||
| } | ||
| }); | ||
|
|
||
| o('should list recursive:false ', async () => { | ||
| const files = await fsa.toArray(fsa.list(prefix, { recursive: false })); | ||
| o(files.length).equals(6); | ||
| o(files.map((f) => f.slice(prefix.length)).map(removeSlashes)).deepEquals([ | ||
| 'a', | ||
| 'c', | ||
| 'd', | ||
| 'file-1', | ||
| 'file-2', | ||
| '🦄.json', | ||
| ]); | ||
| }); | ||
|
|
||
| o('should list folders', async () => { | ||
| const files = await fsa.toArray(fsa.details(prefix, { recursive: false })); | ||
| o( | ||
| files | ||
| .filter((f) => f.isDirectory) | ||
| .map((f) => f.path.slice(prefix.length)) | ||
| .map(removeSlashes), | ||
| ).deepEquals(['a', 'c', 'd']); | ||
| }); | ||
|
|
||
| o('should read a file', async () => { | ||
| const file = await fsa.read(fsa.join(prefix, TestFiles[0].path)); | ||
| o(file.toString()).equals(TestFiles[0].buffer.toString()); | ||
| }); | ||
|
|
||
| o('should head a file', async () => { | ||
| const ret = await fsa.head(fsa.join(prefix, TestFiles[0].path)); | ||
| o(ret.path).equals(fsa.join(prefix, TestFiles[0].path)); | ||
| o(ret.size).equals(TestFiles[0].buffer.length); | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| testPrefix('/tmp/blacha-chunkd-test/'); | ||
| // testPrefix('s3://blacha-chunkd-test/v2/'); | ||
| // testPrefix('s3://blacha-chunkd-test/v3/'); | ||
| // testPrefix('gs://blacha-chunkd-test/'); | ||
|
|
||
| o.run(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| /* eslint-disable @typescript-eslint/no-non-null-assertion */ | ||
| import o from 'ospec'; | ||
| import path from 'path'; | ||
| import 'source-map-support/register.js'; | ||
| import { fileURLToPath } from 'url'; | ||
| import { FsFile } from '../file.fs.js'; | ||
|
|
||
| const __dirname = path.dirname(fileURLToPath(import.meta.url)); | ||
|
|
||
| async function toArray<T>(generator: AsyncGenerator<T>): Promise<T[]> { | ||
| const output: T[] = []; | ||
| for await (const o of generator) output.push(o); | ||
| return output; | ||
| } | ||
|
|
||
| o.spec('LocalFileSystem', () => { | ||
| const fs = new FsFile(); | ||
|
|
||
| o('should read a file', async () => { | ||
| const buf = await fs.read(path.join(__dirname, 'fs.file.test.js')); | ||
| o(buf.toString().includes("import o from 'ospec'")).equals(true); | ||
| }); | ||
|
|
||
| o('should 404 when file not found', async () => { | ||
| try { | ||
| await fs.read(path.join(__dirname, 'NOT A FILE.js')); | ||
| o(true).equals(false); // should have errored | ||
| } catch (e: any) { | ||
| o(e.code).equals(404); | ||
| } | ||
| }); | ||
|
|
||
| o('should head/exists a file', async () => { | ||
| const ref = await fs.head(path.join(__dirname, 'fs.file.test.js')); | ||
| o(ref).notEquals(null); | ||
| }); | ||
|
|
||
| o('should list files', async () => { | ||
| const files = await toArray(fs.list(__dirname)); | ||
|
|
||
| o(files.length > 3).equals(true); | ||
| o(files.find((f) => f.endsWith('__test__/fs.file.test.js'))).notEquals(undefined); | ||
| }); | ||
|
|
||
| o('should list files with details', async () => { | ||
| const files = await toArray(fs.details(__dirname)); | ||
|
|
||
| o(files.length > 3).equals(true); | ||
| o(files.find((f) => f.path.endsWith('__test__/fs.file.test.js'))).notEquals(undefined); | ||
| o(files.filter((f) => f.isDirectory)).deepEquals([]); | ||
| }); | ||
|
|
||
| o('should list recursively', async () => { | ||
| const files = await toArray(fs.details(path.join(__dirname, '..'))); | ||
| o(files.find((f) => f.path.endsWith('__test__/fs.file.test.js'))).notEquals(undefined); | ||
| o(files.filter((f) => f.isDirectory)).deepEquals([]); | ||
| }); | ||
|
|
||
| o('should list folders when not recursive', async () => { | ||
| const files = await toArray(fs.details(path.join(__dirname, '..'), { recursive: false })); | ||
| // In a sub folder shouldn't find it | ||
| o(files.find((f) => f.path.endsWith('__test__/fs.file.test.js'))).equals(undefined); | ||
| o(files.filter((f) => f.isDirectory).length).deepEquals(1); | ||
| }); | ||
| }); |
Oops, something went wrong.