From f0f89decd5fc4aae64df6d2d6fadadaf08a7ef7a Mon Sep 17 00:00:00 2001 From: Kit-p Date: Sun, 29 Jan 2023 08:22:25 +0000 Subject: [PATCH] feat: stringify (compress) --- package.json | 4 +- pnpm-lock.yaml | 19 +++++++++ src/stringify.ts | 31 +++++++++++++- test/stringify.perf.test.ts | 81 ++++++++++++++++++++++++------------- test/stringify.test.ts | 56 +++++++++++++++++++++++++ test/util/format.ts | 23 +++++++++++ test/util/index.ts | 2 + test/util/timer.ts | 27 +++++++++++++ 8 files changed, 211 insertions(+), 32 deletions(-) create mode 100644 test/util/format.ts create mode 100644 test/util/index.ts create mode 100644 test/util/timer.ts diff --git a/package.json b/package.json index 0a06f05..269ef3f 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,8 @@ "packageManager": "pnpm@7.17.0", "dependencies": { "bson": "^4.7.0", - "lodash.clonedeep": "^4.5.0" + "lodash.clonedeep": "^4.5.0", + "lz-string": "^1.4.4" }, "devDependencies": { "@commitlint/cli": "^17.3.0", @@ -68,6 +69,7 @@ "@swc/jest": "^0.2.23", "@types/jest": "^29.2.3", "@types/lodash.clonedeep": "^4.5.7", + "@types/lz-string": "^1.3.34", "@typescript-eslint/eslint-plugin": "^5.44.0", "@typescript-eslint/parser": "^5.44.0", "concurrently": "^7.6.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad6a49c..a1f821e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,6 +13,7 @@ specifiers: "@swc/jest": ^0.2.23 "@types/jest": ^29.2.3 "@types/lodash.clonedeep": ^4.5.7 + "@types/lz-string": ^1.3.34 "@typescript-eslint/eslint-plugin": ^5.44.0 "@typescript-eslint/parser": ^5.44.0 bson: ^4.7.0 @@ -21,6 +22,7 @@ specifiers: husky: ^8.0.2 jest: ^29.3.1 lodash.clonedeep: ^4.5.0 + lz-string: ^1.4.4 prettier: ^2.8.0 pretty-quick: ^3.1.3 rollup: ^3.4.0 @@ -33,6 +35,7 @@ specifiers: dependencies: bson: 4.7.0 lodash.clonedeep: 4.5.0 + lz-string: 1.4.4 devDependencies: "@commitlint/cli": 17.3.0_@swc+core@1.3.21 @@ -47,6 +50,7 @@ devDependencies: "@swc/jest": 0.2.23_@swc+core@1.3.21 "@types/jest": 29.2.3 "@types/lodash.clonedeep": 4.5.7 + "@types/lz-string": 1.3.34 "@typescript-eslint/eslint-plugin": 5.44.0_fnsv2sbzcckq65bwfk7a5xwslu "@typescript-eslint/parser": 5.44.0_hsf322ms6xhhd4b5ne6lb74y4a concurrently: 7.6.0 @@ -1893,6 +1897,13 @@ packages: } dev: true + /@types/lz-string/1.3.34: + resolution: + { + integrity: sha512-j6G1e8DULJx3ONf6NdR5JiR2ZY3K3PaaqiEuKYkLQO0Czfi1AzrtjfnfCROyWGeDd5IVMKCwsgSmMip9OWijow==, + } + dev: true + /@types/minimatch/3.0.5: resolution: { @@ -5212,6 +5223,14 @@ packages: yallist: 4.0.0 dev: true + /lz-string/1.4.4: + resolution: + { + integrity: sha512-0ckx7ZHRPqb0oUm8zNr+90mtf9DQB60H1wMCjBtfi62Kl3a7JbHob6gA2bC+xRvZoOL+1hzUK8jeuEIQE8svEQ==, + } + hasBin: true + dev: false + /magic-string/0.26.7: resolution: { diff --git a/src/stringify.ts b/src/stringify.ts index cb46467..0d328c1 100644 --- a/src/stringify.ts +++ b/src/stringify.ts @@ -1,5 +1,6 @@ import { EJSON } from 'bson'; import cloneDeep from 'lodash.clonedeep'; +import { compressToUTF16 } from 'lz-string'; import { MINIFY_REMAINING_CANDIDATES, MINIFY_STARTING_CANDIDATES, @@ -10,16 +11,19 @@ export type StringifyReplacer = (this: any, key: string, value: any) => any; export interface StringifyOptions { extended?: boolean | { enable: boolean; relaxed?: boolean }; minify?: false | { whitespace?: boolean; key?: boolean }; + compress?: boolean | { enable: boolean }; } interface _StringifyOptions { extended: { enable: boolean; relaxed: boolean }; minify: { whitespace: boolean; key: boolean }; + compress: { enable: boolean }; } const defaultOptions: _StringifyOptions = { extended: { enable: false, relaxed: true }, minify: { whitespace: false, key: false }, + compress: { enable: false }, }; function mergeWithDefaultOptions(input?: StringifyOptions): _StringifyOptions { @@ -53,6 +57,18 @@ function mergeWithDefaultOptions(input?: StringifyOptions): _StringifyOptions { }; } + if (input.compress == null) { + input.compress = defaultOptions.compress; + } else if (typeof input.compress === 'boolean') { + input.compress = { + enable: input.compress, + }; + } else { + input.compress = { + enable: input.compress.enable, + }; + } + return input as _StringifyOptions; } @@ -80,12 +96,19 @@ export function stringify( obj = minifyJsonKeys(obj); } + let result: string; if (_options.extended.enable) { - return EJSON.stringify(obj, _replacer, space, { + result = EJSON.stringify(obj, _replacer, space, { relaxed: _options.extended.relaxed, }); + } else { + result = JSON.stringify(obj, _replacer, space); + } + + if (_options.compress.enable) { + return compressString(result); } - return JSON.stringify(obj, _replacer, space); + return result; } export interface KeyMinifiedJson { @@ -175,3 +198,7 @@ function minifyAllKeys(obj: any, reverseKeyMap: Record): any { return obj; } + +export function compressString(str: string): string { + return compressToUTF16(str); +} diff --git a/test/stringify.perf.test.ts b/test/stringify.perf.test.ts index 8f00401..bc3d6b2 100644 --- a/test/stringify.perf.test.ts +++ b/test/stringify.perf.test.ts @@ -1,44 +1,67 @@ -import { describe, expect, it } from '@jest/globals'; +import { describe, it } from '@jest/globals'; import { JsonKit } from '../src'; +import { formatDuration, timer } from './util'; -describe('[stringify] performance', () => { - it('stringify with ~650KB json data', async () => { - const data = await import('./dataset/ne_110m_populated_places.json'); +const test = (data: any): void => { + const [original, originalDuration] = timer(() => JSON.stringify(data)); - const original = JSON.stringify(data); + const [basic, basicDuration] = timer(() => JsonKit.stringify(data)); - const basic = JsonKit.stringify(data); + const [minified, minifiedDuration] = timer(() => + JsonKit.stringify(data, { minify: { key: true } }) + ); - const minified = JsonKit.stringify(data, { minify: { key: true } }); + const [compressed, compressedDuration] = timer(() => + JsonKit.stringify(data, { compress: true }) + ); - expect(basic).toEqual(original); + const [minifiedAndCompressed, minifiedAndCompressedDuration] = timer(() => + JsonKit.stringify(data, { + minify: { key: true }, + compress: true, + }) + ); - console.info( - `${original.length} -> ${minified.length} = ${( - ((minified.length - original.length) / original.length) * - 100 - ).toFixed(2)}%` - ); - expect(minified.length).toBeLessThan(original.length); - }); + console.info( + `baseline: ${original.length} (±0.00%) [${formatDuration( + originalDuration + )}]` + ); - it('stringify with ~50MB json data', async () => { - const data = await import('./dataset/ne_10m_roads.json'); + console.info( + `basic: ${basic.length} (±0.00%) [${formatDuration(basicDuration)}]` + ); - const original = JSON.stringify(data); + console.info( + `minify: ${minified.length} (${( + ((minified.length - original.length) / original.length) * + 100 + ).toFixed(2)}%) [${formatDuration(minifiedDuration)}]` + ); - const basic = JsonKit.stringify(data); + console.info( + `compress: ${compressed.length} (${( + ((compressed.length - original.length) / original.length) * + 100 + ).toFixed(2)}%) [${formatDuration(compressedDuration)}]` + ); - const minified = JsonKit.stringify(data, { minify: { key: true } }); + console.info( + `minify + compress: ${minifiedAndCompressed.length} (${( + ((minifiedAndCompressed.length - original.length) / original.length) * + 100 + ).toFixed(2)}%) [${formatDuration(minifiedAndCompressedDuration)}]` + ); +}; - expect(basic).toEqual(original); +describe('[stringify] performance', () => { + it('stringify with ~650KB json data', async () => { + const data = await import('./dataset/ne_110m_populated_places.json'); + test(data); + }); - console.info( - `${original.length} -> ${minified.length} = ${( - ((minified.length - original.length) / original.length) * - 100 - ).toFixed(2)}%` - ); - expect(minified.length).toBeLessThan(original.length); + it('stringify with ~50MB json data', async () => { + const data = await import('./dataset/ne_10m_roads.json'); + test(data); }); }); diff --git a/test/stringify.test.ts b/test/stringify.test.ts index 8ac07d1..f0b8927 100644 --- a/test/stringify.test.ts +++ b/test/stringify.test.ts @@ -52,18 +52,22 @@ describe('[stringify] basic', () => { const optionsShorthand = JsonKit.stringify(obj, { extended: true, minify: false, + compress: true, }); const optionsFull = JsonKit.stringify(obj, { extended: { enable: false }, minify: { whitespace: false, key: false }, + compress: { enable: false }, }); const optionsDefaultFull = JsonKit.stringify(obj, { extended: { enable: true, relaxed: true }, minify: { whitespace: false, key: false }, + compress: { enable: true }, }); const defaultOptions = JsonKit.stringify(obj, { extended: { enable: false, relaxed: true }, minify: { whitespace: false, key: false }, + compress: { enable: false }, }); expect(base).toEqual(defaultOptions); @@ -167,3 +171,55 @@ describe('[stringify] minify', () => { ); }); }); + +describe('[stringify] compress', () => { + const obj = { + a: 1, + '2': '2', + '': true, + ' ': null, + 'a-b': undefined, + _c: new Date(), + toString: () => 'string', + arr: [1, '2', true, null, undefined, new Date(), () => 'string', [], {}], + nested: { + a: 1, + '2': '2', + }, + empty: {}, + }; + + it('stringify with compression should have identical behavior as built-in JSON.stringify and compression', () => { + const replacer: StringifyReplacer = () => 'test'; + + expect(JsonKit.stringify(obj, { compress: true })).toEqual( + JsonKit.compressString(JSON.stringify(obj)) + ); + + expect(JsonKit.stringify(obj, replacer, ' ', { compress: true })).toEqual( + JsonKit.compressString(JSON.stringify(obj, replacer, ' ')) + ); + }); + + it('stringify with compression should have identical behavior as bson.EJSON.stringify and compression', () => { + const replacer: StringifyReplacer = () => 'test'; + + expect( + JsonKit.stringify(obj, { + extended: { enable: true, relaxed: false }, + compress: { enable: true }, + }) + ).toEqual(JsonKit.compressString(EJSON.stringify(obj, { relaxed: false }))); + + expect( + JsonKit.stringify(obj, { + extended: { enable: true, relaxed: true }, + compress: { enable: true }, + }) + ).toEqual(JsonKit.compressString(EJSON.stringify(obj, { relaxed: true }))); + + expect( + JsonKit.stringify(obj, replacer, 4, { extended: true, compress: true }) + ).toEqual(JsonKit.compressString(EJSON.stringify(obj, replacer, 4))); + }); +}); diff --git a/test/util/format.ts b/test/util/format.ts new file mode 100644 index 0000000..44959e6 --- /dev/null +++ b/test/util/format.ts @@ -0,0 +1,23 @@ +export function formatDuration(milliseconds: number) { + if (milliseconds < 1000) { + return `${milliseconds.toFixed(0)} ms`; + } + + const seconds = milliseconds / 1000; + if (seconds < 60) { + return `${seconds.toFixed(3)} s`; + } + + const minutes = seconds / 60; + if (minutes < 60) { + return `${minutes.toFixed(2)} m`; + } + + const hours = minutes / 60; + if (hours < 24) { + return `${hours.toFixed(2)} h`; + } + + const days = hours / 24; + return `${days.toFixed(2)} d`; +} diff --git a/test/util/index.ts b/test/util/index.ts new file mode 100644 index 0000000..90bf6a8 --- /dev/null +++ b/test/util/index.ts @@ -0,0 +1,2 @@ +export * from './format'; +export * from './timer'; diff --git a/test/util/timer.ts b/test/util/timer.ts new file mode 100644 index 0000000..6dcd5fa --- /dev/null +++ b/test/util/timer.ts @@ -0,0 +1,27 @@ +/** + * @param target target function to measure + * @returns Array of [result, durationInMs] + */ +export function timer(target: () => T): [T, number]; +export async function timer>( + target: () => T +): Promise<[Awaited, number]>; +export function timer( + target: () => any +): [any, number] | Promise<[any, number]> { + const start = performance.now(); + const result = target(); + if (typeof result?.then === 'function') { + return new Promise((resolve, reject) => { + result + .then((awaited: any) => { + const durationInMs = performance.now() - start; + return resolve([awaited, durationInMs]); + }) + .catch((error: any) => reject(error)); + }); + } else { + const durationInMs = performance.now() - start; + return [result, durationInMs]; + } +}