Skip to content

Commit

Permalink
Added a way to silently ignore JSON parsing errors.
Browse files Browse the repository at this point in the history
  • Loading branch information
uhop committed Sep 28, 2024
1 parent f6c6a86 commit 8061f7e
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 11 deletions.
11 changes: 9 additions & 2 deletions src/jsonl/parser.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,19 @@ interface OutputItem {
*/
type Reviver = (this: unknown, key: string, value: unknown) => unknown;

type ParserOptions = {
/** An optional reviver function for `JSON.parse()`. */
reviver?: Reviver;
/** Whether to ignore errors silently. It defaults to `false`. */
ignoreErrors?: boolean;
};

/**
* The JSONL parser as a streamable generator.
* @param reviver an optional reviver function (see {@link Reviver})
* @param reviver an optional reviver function (see {@link Reviver}) or an {@link ParserOptions}
* @returns an asynchronous generator
* @remark parsers JSON lines items returning them as {@link OutputItem}.
*/
declare function parser(
reviver?: Reviver
reviver?: Reviver | ParserOptions
): (x: string | Buffer) => AsyncGenerator<OutputItem, void, unknown>;
19 changes: 17 additions & 2 deletions src/jsonl/parser.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,28 @@

'use strict';

const {none} = require('../defs.js');
const gen = require('../gen.js');
const fixUtf8Stream = require('../utils/fixUtf8Stream');
const lines = require('../utils/lines');

const parser = reviver => {
const parse = reviver => string => JSON.parse(string, reviver);

const checkedParse = reviver => string => {
try {
return JSON.parse(string, reviver);
} catch (_) {
// squelch
return none;
}
};

const parser = options => {
const reviver = (options && options.reviver) || options,
ignoreErrors = options && options.ignoreErrors,
parseFn = ignoreErrors ? checkedParse(reviver) : parse(reviver);
let counter = 0;
return gen(fixUtf8Stream(), lines(), string => ({key: counter++, value: JSON.parse(string, reviver)}));
return gen(fixUtf8Stream(), lines(), string => ({key: counter++, value: parseFn(string)}));
};

module.exports = parser;
2 changes: 2 additions & 0 deletions src/jsonl/parserStream.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ interface ParserOptions extends DuplexOptions {
* @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse
*/
reviver?: (this: unknown, key: string, value: unknown) => unknown;
/** Whether to ignore errors silently. It defaults to `false`. */
ignoreErrors?: boolean;
}

/**
Expand Down
10 changes: 4 additions & 6 deletions src/jsonl/parserStream.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,14 @@

'use strict';

const gen = require('../gen');
const asStream = require('../asStream');
const fixUtf8Stream = require('../utils/fixUtf8Stream');
const lines = require('../utils/lines');
const parser = require('./parser.js');

const parserStream = options => {
const reviver = options && options.reviver;
let counter = 0;
const reviver = options && options.reviver,
ignoreErrors = options && options.ignoreErrors;
return asStream(
gen(fixUtf8Stream(), lines(), string => ({key: counter++, value: JSON.parse(string, reviver)})),
parser({reviver, ignoreErrors}),
Object.assign({writableObjectMode: false, readableObjectMode: true}, options)
);
};
Expand Down
2 changes: 1 addition & 1 deletion wiki
Submodule wiki updated from edbfec to daa110

0 comments on commit 8061f7e

Please sign in to comment.