feat(group): create a output file per item grouped (#565)

linz · Jul 25, 2023 · 5d77c40 · 5d77c40
1 parent 8c4ed41
commit 5d77c40
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 14 deletions.
diff --git a/src/commands/group/__test__/group.test.ts b/src/commands/group/__test__/group.test.ts
@@ -25,12 +25,15 @@ describe('groupItems', () => {
 
 describe('group', () => {
  const memoryFs = new FsMemory();
+
  before(() => {
  fsa.register('/tmp/group', memoryFs);
  });
+
  it('should load from a JSON array', async () => {
  await commandGroup.handler({ inputs: [JSON.stringify([1, 2, 3, 4])], forceOutput: true, size: 50 } as any);
- assert.deepEqual(await fsa.readJson('/tmp/group/output.json'), [[1, 2, 3, 4]]);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output.json'), ['000']);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output/000.json'), [1, 2, 3, 4]);
  });
 
  it('should load from multiple JSON arrays', async () => {
@@ -39,21 +42,21 @@ describe('group', () => {
  forceOutput: true,
  size: 3,
  } as any);
- assert.deepEqual(await fsa.readJson('/tmp/group/output.json'), [
-  [1, 2, 3],
-  [4, 'alpha'],
- ]);
+
+ assert.deepEqual(await fsa.readJson('/tmp/group/output.json'), ['000', '001']);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output/000.json'), [1, 2, 3]);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output/001.json'), [4, 'alpha']);
  });
+
  it('should load from strings', async () => {
  await commandGroup.handler({
  inputs: ['s3://foo/bar', JSON.stringify([1, 2, 3, 4]), JSON.stringify(['alpha'])],
  forceOutput: true,
  size: 3,
  } as any);
- assert.deepEqual(await fsa.readJson('/tmp/group/output.json'), [
- ['s3://foo/bar', 1, 2],
- [3, 4, 'alpha'],
- ]);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output.json'), ['000', '001']);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output/000.json'), ['s3://foo/bar', 1, 2]);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output/001.json'), [3, 4, 'alpha']);
  });
 
  it('should load from a file', async () => {
@@ -64,9 +67,8 @@ describe('group', () => {
  forceOutput: true,
  size: 3,
  } as any);
- assert.deepEqual(await fsa.readJson('/tmp/group/output.json'), [
- [1, 2, 3],
- [4, 5],
- ]);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output.json'), ['000', '001']);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output/000.json'), [1, 2, 3]);
+ assert.deepEqual(await fsa.readJson('/tmp/group/output/001.json'), [4, 5]);
  });
 });
diff --git a/src/commands/group/group.ts b/src/commands/group/group.ts
@@ -73,7 +73,16 @@ export const commandGroup = command({
  const grouped = groupItems(inputs, args.size);
  logger.info({ files: inputs.length, groups: grouped.length }, 'Group:Done');
  if (args.forceOutput || isArgo()) {
- await fsa.write('/tmp/group/output.json', JSON.stringify(grouped));
+ const items = [];
+ // Write out a file per group into /tmp/group/output/000.json
+ for (let i = 0; i < grouped.length; i++) {
+ const groupId = String(i).padStart(3, '0');
+ await fsa.write(`/tmp/group/output/${groupId}.json`, JSON.stringify(grouped[i], null, 2));
+ items.push(groupId);
+ }
+
+ // output.json contains ["001","002","003","004","005","006","007"...]
+ await fsa.write('/tmp/group/output.json', JSON.stringify(items));
  }
  },
 });