From d4c1883fa6abd4dea7d76eb59221d2fc917935eb Mon Sep 17 00:00:00 2001 From: Zihua Li Date: Wed, 7 Jun 2023 07:59:55 +0800 Subject: [PATCH] Add fuzzing tests --- .gitignore | 6 +- _develop/karma.config.js | 1 + _develop/webpack.config.js | 1 + package.json | 5 +- test/fuzzing.ts | 2 + test/fuzzing/applyDelta.test.ts | 224 ++++++++++++++++++ .../{random.ts => fuzzing/tableEmbed.test.ts} | 54 ++--- test/fuzzing/utils.ts | 7 + 8 files changed, 265 insertions(+), 35 deletions(-) create mode 100644 test/fuzzing.ts create mode 100644 test/fuzzing/applyDelta.test.ts rename test/{random.ts => fuzzing/tableEmbed.test.ts} (75%) create mode 100644 test/fuzzing/utils.ts diff --git a/.gitignore b/.gitignore index 862c9515f8..137de48830 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,11 @@ formats/*.js modules/*.js themes/*.js ui/*.js -test/random.js + +test/**/*.js +!test/helpers/**/*.js +!test/unit/**/*.js +!test/unit.js core.js quill.js diff --git a/_develop/karma.config.js b/_develop/karma.config.js index 381528a912..ed9e76666b 100644 --- a/_develop/karma.config.js +++ b/_develop/karma.config.js @@ -15,6 +15,7 @@ module.exports = config => { }, { pattern: 'dist/quill.snow.css', nocache: true }, { pattern: 'dist/unit.js', nocache: true }, + { pattern: 'dist/fuzzing.js', nocache: true }, { pattern: 'dist/*.map', included: false, served: true, nocache: true }, { pattern: 'assets/favicon.png', included: false, served: true }, ], diff --git a/_develop/webpack.config.js b/_develop/webpack.config.js index f8b02c75dd..08b4d13a6e 100644 --- a/_develop/webpack.config.js +++ b/_develop/webpack.config.js @@ -65,6 +65,7 @@ const baseConfig = { 'quill.bubble': './assets/bubble.styl', 'quill.snow': './assets/snow.styl', 'unit.js': './test/unit.js', + 'fuzzing.js': './test/fuzzing.ts', }, output: { filename: '[name]', diff --git a/package.json b/package.json index 12ade12fd7..c87e24b571 100644 --- a/package.json +++ b/package.json @@ -99,12 +99,11 @@ "website:build": "npm run build -w website", "website:serve": "npm run serve -w website -- --port $npm_package_config_ports_gatsby", "website:develop": "npm run develop -w website -- --port $npm_package_config_ports_gatsby", - "test": "npm run test:unit; npm run test:random", - "test:all": "npm run test:unit; npm run test:functional; npm run test:random", + "test": "npm run test:unit", + "test:all": "npm run test:unit; npm run test:functional", "test:e2e": "npx playwright test", "test:unit": "npm run build; karma start _develop/karma.config.js", "test:unit:ci": "npm run build; karma start _develop/karma.config.js --reporters dots,saucelabs", - "test:random": "ts-node --preferTsExts -O '{\"module\":\"commonjs\"}' ./node_modules/.bin/jasmine test/random.ts", "test:coverage": "webpack --env.coverage --config _develop/webpack.config.js; karma start _develop/karma.config.js --reporters coverage" }, "overrides": { diff --git a/test/fuzzing.ts b/test/fuzzing.ts new file mode 100644 index 0000000000..0c3f721e68 --- /dev/null +++ b/test/fuzzing.ts @@ -0,0 +1,2 @@ +import './fuzzing/applyDelta.test'; +import './fuzzing/tableEmbed.test'; diff --git a/test/fuzzing/applyDelta.test.ts b/test/fuzzing/applyDelta.test.ts new file mode 100644 index 0000000000..76aa622743 --- /dev/null +++ b/test/fuzzing/applyDelta.test.ts @@ -0,0 +1,224 @@ +import Delta, { AttributeMap, Op } from 'quill-delta'; +import { choose, randomInt } from './utils'; +import { AlignClass } from '../../formats/align'; +import { FontClass } from '../../formats/font'; +import { SizeClass } from '../../formats/size'; +import Quill from '../../quill'; + +type AttributeDef = { name: string; values: (number | string | boolean)[] }; +const BLOCK_EMBED_NAME = 'video'; +const INLINE_EMBED_NAME = 'image'; + +const attributeDefs: { + text: AttributeDef[]; + newline: AttributeDef[]; + inlineEmbed: AttributeDef[]; + blockEmbed: AttributeDef[]; +} = { + text: [ + { name: 'color', values: ['#ffffff', '#000000', '#ff0000', '#ffff00'] }, + { name: 'bold', values: [true] }, + { name: 'code', values: [true] }, + // @ts-expect-error + { name: 'font', values: FontClass.whitelist }, + // @ts-expect-error + { name: 'size', values: SizeClass.whitelist }, + ], + newline: [ + // @ts-expect-error + { name: 'align', values: AlignClass.whitelist }, + { name: 'header', values: [1, 2, 3, 4, 5] }, + { name: 'blockquote', values: [true] }, + { name: 'list', values: ['ordered', 'bullet', 'checked', 'unchecked'] }, + ], + inlineEmbed: [ + { name: 'width', values: ['100', '200', '300'] }, + { name: 'height', values: ['100', '200', '300'] }, + ], + blockEmbed: [ + { name: 'width', values: ['100', '200', '300'] }, + { name: 'height', values: ['100', '200', '300'] }, + ], +}; + +const isLineFinished = (delta: Delta) => { + const lastOp = delta.ops[delta.ops.length - 1]; + if (!lastOp) return false; + if (typeof lastOp.insert === 'string') { + return lastOp.insert.endsWith('\n'); + } + if (typeof lastOp.insert === 'object') { + const key = Object.keys(lastOp.insert)[0]; + return key === BLOCK_EMBED_NAME; + } + throw new Error('invalid op'); +}; + +const generateAttributes = (scope: keyof typeof attributeDefs) => { + const attributeCount = + scope === 'newline' + ? // Some block-level formats are exclusive so we only pick one for now for simplicity + choose([0, 0, 1]) + : choose([0, 0, 0, 0, 0, 1, 2, 3, 4]); + const attributes: AttributeMap = {}; + for (let i = 0; i < attributeCount; i += 1) { + const def = choose(attributeDefs[scope]); + attributes[def.name] = choose(def.values); + } + return attributes; +}; + +const generateRandomText = () => { + return choose([ + 'hi', + 'world', + 'Slab', + ' ', + 'this is a long text that contains spaces', + ]); +}; + +type SingleInsertValue = + | string + | { [INLINE_EMBED_NAME]: string } + | { [BLOCK_EMBED_NAME]: string }; + +const generateSingleInsertDelta = (): Delta['ops'][number] & { + insert: SingleInsertValue; +} => { + const operation = choose([ + 'text', + 'text', + 'text', + 'newline', + 'inlineEmbed', + 'blockEmbed', + ]); + + let insert: SingleInsertValue; + switch (operation) { + case 'text': + insert = generateRandomText(); + break; + case 'newline': + insert = '\n'; + break; + case 'inlineEmbed': + insert = { [INLINE_EMBED_NAME]: 'https://example.com' }; + break; + case 'blockEmbed': { + insert = { [BLOCK_EMBED_NAME]: 'https://example.com' }; + break; + } + } + + const attributes = generateAttributes(operation); + const op: Op & { insert: SingleInsertValue } = { insert }; + if (Object.keys(attributes).length) { + op.attributes = attributes; + } + return op; +}; + +const safePushInsert = (delta: Delta) => { + const op = generateSingleInsertDelta(); + if (typeof op.insert === 'object' && op.insert[BLOCK_EMBED_NAME]) { + delta.insert('\n'); + } + delta.push(op); +}; + +const generateDocument = () => { + const delta = new Delta(); + const operationCount = 2 + randomInt(20); + for (let i = 0; i < operationCount; i += 1) { + safePushInsert(delta); + } + return delta; +}; + +const generateChange = (doc: Delta, changeCount: number) => { + const docLength = doc.length(); + const skipLength = randomInt(docLength); + let change = new Delta().retain(skipLength); + const action = choose(['insert', 'delete', 'retain']); + const nextOp = doc.slice(skipLength).ops[0]; + if (!nextOp) throw new Error('nextOp expected'); + const needNewline = !isLineFinished(doc.slice(0, skipLength)); + switch (action) { + case 'insert': { + const delta = new Delta(); + const operationCount = randomInt(5) + 1; + for (let i = 0; i < operationCount; i += 1) { + safePushInsert(delta); + } + if ( + needNewline || + (typeof nextOp.insert === 'object' && !!nextOp.insert[BLOCK_EMBED_NAME]) + ) { + delta.insert('\n'); + } + change = change.concat(delta); + break; + } + case 'delete': { + const lengthToDelete = randomInt(docLength - skipLength - 1) + 1; + const nextOpAfterDelete = doc.slice(skipLength + lengthToDelete).ops[0]; + if ( + needNewline && + (!nextOpAfterDelete || + (typeof nextOpAfterDelete.insert === 'object' && + !!nextOpAfterDelete.insert[BLOCK_EMBED_NAME])) + ) { + change.insert('\n'); + } + change.delete(lengthToDelete); + break; + } + case 'retain': { + const retainLength = + typeof nextOp.insert === 'string' + ? randomInt(nextOp.insert.length - 1) + 1 + : 1; + if (typeof nextOp.insert === 'string') { + if ( + nextOp.insert.includes('\n') && + nextOp.insert.replace(/\n/g, '').length + ) { + break; + } + if (nextOp.insert.includes('\n')) { + change.retain( + retainLength, + AttributeMap.diff(nextOp.attributes, generateAttributes('newline')), + ); + } else { + change.retain( + retainLength, + AttributeMap.diff(nextOp.attributes, generateAttributes('text')), + ); + } + break; + } + break; + } + } + changeCount -= 1; + return changeCount <= 0 + ? change + : change.compose(generateChange(doc.compose(change), changeCount)); +}; + +describe('applyDelta', () => { + it('random', () => { + const container = document.createElement('div'); + const quill = new Quill(container); + quill.setContents(generateDocument()); + for (let i = 0; i < 1000; i += 1) { + const doc = quill.getContents(); + const change = generateChange(doc, randomInt(4) + 1); + const diff = quill.updateContents(change); + expect(change).toEqual(diff); + } + }); +}); diff --git a/test/random.ts b/test/fuzzing/tableEmbed.test.ts similarity index 75% rename from test/random.ts rename to test/fuzzing/tableEmbed.test.ts index 46cfda8ec0..f30dc512da 100644 --- a/test/random.ts +++ b/test/fuzzing/tableEmbed.test.ts @@ -3,16 +3,8 @@ import TableEmbed, { CellData, TableData, TableRowColumnOp, -} from '../modules/tableEmbed'; - -// Random testing in order to find unknown issues. - -const random = choices => { - if (typeof choices === 'number') { - return Math.floor(Math.random() * choices); - } - return choices[random(choices.length)]; -}; +} from '../../modules/tableEmbed'; +import { choose, randomInt } from './utils'; const getRandomRowColumnId = () => { const characters = 'abcdefghijklmnopqrstuvwxyz0123456789'; @@ -26,16 +18,16 @@ const attachAttributes = ( obj: T, ): T & { attributes: Record } => { const getRandomAttributes = () => { - const attributeCount = random([1, 4, 8]); + const attributeCount = choose([1, 4, 8]); const allowedAttributes = ['align', 'background', 'color', 'font']; const allowedValues = ['center', 'red', 'left', 'uppercase']; const attributes = {}; new Array(attributeCount).fill(0).forEach(() => { - attributes[random(allowedAttributes)] = random(allowedValues); + attributes[choose(allowedAttributes)] = choose(allowedValues); }); return attributes; }; - if (random([true, false])) { + if (choose([true, false])) { // @ts-expect-error obj.attributes = getRandomAttributes(); } @@ -44,14 +36,14 @@ const attachAttributes = ( }; const getRandomCellContent = () => { - const opCount = random([1, 2, 3]); + const opCount = choose([1, 2, 3]); const delta = new Delta(); new Array(opCount).fill(0).forEach(() => { delta.push( attachAttributes({ - insert: new Array(random(10) + 1) + insert: new Array(randomInt(10) + 1) .fill(0) - .map(() => random(['a', 'b', 'c', 'c', 'e', 'f', 'g'])) + .map(() => choose(['a', 'b', 'c', 'c', 'e', 'f', 'g'])) .join(''), }), ); @@ -67,26 +59,26 @@ const getRandomChange = base => { base.ops[0].insert['table-embed'].columns || [], ).length(), }; - ['rows', 'columns'].forEach(field => { + (['rows', 'columns'] as const).forEach(field => { const baseLength = dimension[field]; - const action = random(['insert', 'delete', 'retain']); + const action = choose(['insert', 'delete', 'retain']); const delta = new Delta(); switch (action) { case 'insert': - delta.retain(random(baseLength + 1)); + delta.retain(randomInt(baseLength + 1)); delta.push( attachAttributes({ insert: { id: getRandomRowColumnId() } }), ); break; case 'delete': if (baseLength >= 1) { - delta.retain(random(baseLength)); + delta.retain(randomInt(baseLength)); delta.delete(1); } break; case 'retain': if (baseLength >= 1) { - delta.retain(random(baseLength)); + delta.retain(randomInt(baseLength)); delta.push(attachAttributes({ retain: 1 })); } break; @@ -98,10 +90,10 @@ const getRandomChange = base => { } }); - const updateCellCount = random([0, 1, 2, 3]); + const updateCellCount = choose([0, 1, 2, 3]); new Array(updateCellCount).fill(0).forEach(() => { - const row = random(dimension.rows); - const column = random(dimension.columns); + const row = randomInt(dimension.rows); + const column = randomInt(dimension.columns); const cellIdentityToModify = `${row + 1}:${column + 1}`; table.cells = { [cellIdentityToModify]: attachAttributes({ @@ -121,9 +113,9 @@ const getRandomRowColumnInsert = (count: number): TableRowColumnOp[] => { }; const getRandomBase = () => { - const rowCount = random([0, 1, 2, 3]); - const columnCount = random([0, 1, 2]); - const cellCount = random([0, 1, 2, 3, 4, 5]); + const rowCount = choose([0, 1, 2, 3]); + const columnCount = choose([0, 1, 2]); + const cellCount = choose([0, 1, 2, 3, 4, 5]); const table: TableData = {}; if (rowCount) table.rows = getRandomRowColumnInsert(rowCount); @@ -131,11 +123,11 @@ const getRandomBase = () => { if (cellCount) { const cells = {}; new Array(cellCount).fill(0).forEach(() => { - const row = random(rowCount); - const column = random(columnCount); + const row = randomInt(rowCount); + const column = randomInt(columnCount); const identity = `${row + 1}:${column + 1}`; const cell: CellData = attachAttributes({}); - if (random([true, false])) { + if (choose([true, false])) { cell.content = getRandomCellContent(); } if (Object.keys(cell).length) { @@ -158,7 +150,7 @@ const runTestCase = () => { ); }; -describe('random tests', () => { +describe('tableEmbed', () => { beforeAll(() => { TableEmbed.register(); }); diff --git a/test/fuzzing/utils.ts b/test/fuzzing/utils.ts new file mode 100644 index 0000000000..652580e2f3 --- /dev/null +++ b/test/fuzzing/utils.ts @@ -0,0 +1,7 @@ +export function randomInt(max: number) { + return Math.floor(Math.random() * max); +} + +export function choose(choices: T[]): T { + return choices[randomInt(choices.length)]; +}