From 1349bb4d02ba7f0f6076f7ff7d0d032585cc4cc0 Mon Sep 17 00:00:00 2001 From: Tim <83622086+timothycohen@users.noreply.github.com> Date: Fri, 26 Jul 2024 06:39:56 -0400 Subject: [PATCH] fix(core): fix decoration offset edge cases (#728) --- packages/core/src/transformer-decorations.ts | 9 ++++ packages/core/src/types/decorations.ts | 2 - packages/core/src/utils.ts | 11 ++++- packages/shiki/test/decorations.test.ts | 47 +++++++++++++++++-- .../shiki/test/out/decorations/basic.html | 2 +- 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/packages/core/src/transformer-decorations.ts b/packages/core/src/transformer-decorations.ts index 76379fb92..2f6ade007 100644 --- a/packages/core/src/transformer-decorations.ts +++ b/packages/core/src/transformer-decorations.ts @@ -21,12 +21,21 @@ export function transformerDecorations(): ShikiTransformer { function normalizePosition(p: OffsetOrPosition): ResolvedPosition { if (typeof p === 'number') { + if (p < 0 || p > shiki.source.length) + throw new ShikiError(`Invalid decoration offset: ${p}. Code length: ${shiki.source.length}`) + return { ...converter.indexToPos(p), offset: p, } } else { + const line = converter.lines[p.line] + if (line === undefined) + throw new ShikiError(`Invalid decoration position ${JSON.stringify(p)}. Lines length: ${converter.lines.length}`) + if (p.character < 0 || p.character > line.length) + throw new ShikiError(`Invalid decoration position ${JSON.stringify(p)}. Line ${p.line} length: ${line.length}`) + return { ...p, offset: converter.posToIndex(p.line, p.character), diff --git a/packages/core/src/types/decorations.ts b/packages/core/src/types/decorations.ts index 50b99c335..34c6272fb 100644 --- a/packages/core/src/types/decorations.ts +++ b/packages/core/src/types/decorations.ts @@ -14,8 +14,6 @@ export interface DecorationItem { start: OffsetOrPosition /** * End offset or position of the decoration. - * - * If the */ end: OffsetOrPosition /** diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 3decf1452..4a28d81aa 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -7,7 +7,7 @@ export function toArray(x: MaybeArray): T[] { } /** - * Slipt a string into lines, each line preserves the line ending. + * Split a string into lines, each line preserves the line ending. */ export function splitLines(code: string, preserveEnding = false): [string, number][] { const parts = code.split(/(\r?\n)/g) @@ -192,11 +192,20 @@ export function stringifyTokenStyle(token: Record) { /** * Creates a converter between index and position in a code block. + * + * Overflow/underflow are unchecked. */ export function createPositionConverter(code: string) { const lines = splitLines(code, true).map(([line]) => line) function indexToPos(index: number): Position { + if (index === code.length) { + return { + line: lines.length - 1, + character: lines[lines.length - 1].length, + } + } + let character = index let line = 0 for (const lineText of lines) { diff --git a/packages/shiki/test/decorations.test.ts b/packages/shiki/test/decorations.test.ts index 61b6d8ab2..1ffc0c0d5 100644 --- a/packages/shiki/test/decorations.test.ts +++ b/packages/shiki/test/decorations.test.ts @@ -23,7 +23,7 @@ export function codeToHtml( let result = hastToHtml(codeToHast(internal, code, options, context)) return result } -` +// final` describe('decorations', () => { it('works', async () => { @@ -78,6 +78,13 @@ describe('decorations', () => { end: { line: 8, character: 25 }, properties: { class: 'highlighted' }, }, + // "// final" + // Testing offset === code.length edge case + { + start: code.length - 8, + end: code.length, + properties: { class: 'highlighted' }, + }, ], }) @@ -126,7 +133,7 @@ describe('decorations errors', () => { ], }) }).rejects - .toThrowErrorMatchingInlineSnapshot(`[TypeError: Cannot read properties of undefined (reading 'length')]`) + .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":100,"character":0}. Lines length: 12]`) }) it('throws when chars overflow', async () => { @@ -139,6 +146,40 @@ describe('decorations errors', () => { ], }) }).rejects - .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Failed to find end index for decoration {"line":0,"character":10,"offset":10}]`) + .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":0,"character":10}. Line 0 length: 4]`) + + expect(async () => { + await codeToHtml(code, { + theme: 'vitesse-light', + lang: 'ts', + decorations: [ + { + start: { line: 2, character: 1 }, + end: { line: 1, character: 36 }, // actual position is { line: 2, character: 3, offset 40 } + }, + ], + }) + }).rejects + .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":1,"character":36}. Line 1 length: 33]`) + }) + + it('throws when offset underflows/overflows', async () => { + expect(async () => { + await codeToHtml(code, { + theme: 'vitesse-light', + lang: 'ts', + decorations: [{ start: 1, end: 1000 }], + }) + }).rejects + .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: 1000. Code length: 252]`) + + expect(async () => { + await codeToHtml(code, { + theme: 'vitesse-light', + lang: 'ts', + decorations: [{ start: -3, end: 5 }], + }) + }).rejects + .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: -3. Code length: 252]`) }) }) diff --git a/packages/shiki/test/out/decorations/basic.html b/packages/shiki/test/out/decorations/basic.html index 6a8a3ad22..324202fc1 100644 --- a/packages/shiki/test/out/decorations/basic.html +++ b/packages/shiki/test/out/decorations/basic.html @@ -18,4 +18,4 @@ let result = hastToHtml(codeToHast(internal, code, options, context)) return result } - \ No newline at end of file +// final \ No newline at end of file