Skip to content

Commit

Permalink
fix: Improve Trace Results (#5133)
Browse files Browse the repository at this point in the history
  • Loading branch information
Jason3S authored Jan 1, 2024
1 parent 27a6d73 commit 1c53140
Show file tree
Hide file tree
Showing 19 changed files with 469 additions and 130 deletions.
38 changes: 26 additions & 12 deletions packages/cspell-lib/api/api.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,26 @@ declare class SpellingDictionaryLoadError extends Error {
}
declare function isSpellingDictionaryLoadError(e: Error): e is SpellingDictionaryLoadError;

type Href = string;
interface DictionaryTraceResult {
word: string;
found: boolean;
foundWord: string | undefined;
forbidden: boolean;
noSuggest: boolean;
dictName: string;
dictSource: string;
configSource: Href | undefined;
errors: Error[] | undefined;
}
interface WordSplits {
word: string;
found: boolean;
}
interface TraceResult$1 extends Array<DictionaryTraceResult> {
splits?: readonly WordSplits[];
}

interface WordSuggestion extends SuggestionResult {
/**
* The suggested word adjusted to match the original case.
Expand Down Expand Up @@ -799,16 +819,7 @@ declare class DocumentValidator {
* @returns MatchRanges of text to include.
*/
getCheckedTextRanges(): MatchRange[];
traceWord(word: string): {
word: string;
found: boolean;
foundWord: string | undefined;
forbidden: boolean;
noSuggest: boolean;
dictName: string;
dictSource: string;
errors: Error[] | undefined;
}[];
traceWord(word: string): TraceResult$1;
private defaultParser;
private _checkParsedText;
private addPossibleError;
Expand Down Expand Up @@ -978,8 +989,11 @@ interface TraceOptions {
ignoreCase?: boolean;
allowCompoundWords?: boolean;
}
interface TraceWordResult extends Array<TraceResult> {
splits: readonly WordSplits[];
}
declare function traceWords(words: string[], settings: CSpellSettings, options: TraceOptions | undefined): Promise<TraceResult[]>;
declare function traceWordsAsync(words: Iterable<string> | AsyncIterable<string>, settings: CSpellSettings, options: TraceOptions | undefined): AsyncIterableIterator<TraceResult[]>;
declare function traceWordsAsync(words: Iterable<string> | AsyncIterable<string>, settings: CSpellSettings, options: TraceOptions | undefined): AsyncIterableIterator<TraceWordResult>;

type Console = typeof console;
interface Logger {
Expand Down Expand Up @@ -1032,4 +1046,4 @@ interface PerfTimer {
type TimeNowFn = () => number;
declare function createPerfTimer(name: string, onEnd?: (elapsed: number, name: string) => void, timeNowFn?: TimeNowFn): PerfTimer;

export { type CheckTextInfo, type ConfigurationDependencies, type CreateTextDocumentParams, type DetermineFinalDocumentSettingsResult, type Document, DocumentValidator, type DocumentValidatorOptions, ENV_CSPELL_GLOB_ROOT, type ExcludeFilesGlobMap, type ExclusionFunction, exclusionHelper_d as ExclusionHelper, type FeatureFlag, FeatureFlags, ImportError, type ImportFileRefWithError$1 as ImportFileRefWithError, IncludeExcludeFlag, type IncludeExcludeOptions, index_link_d as Link, type Logger, type PerfTimer, type SpellCheckFileOptions, type SpellCheckFileResult, SpellingDictionaryLoadError, type SuggestedWord, SuggestionError, type SuggestionOptions, type SuggestionsForWordResult, text_d as Text, type TextDocument, type TextDocumentLine, type TextDocumentRef, type TextInfoItem, type TraceOptions, type TraceResult, UnknownFeatureFlagError, type ValidationIssue, calcOverrideSettings, checkFilenameMatchesGlob, checkText, checkTextDocument, clearCachedFiles, clearCaches, combineTextAndLanguageSettings, combineTextAndLanguageSettings as constructSettingsForText, createConfigLoader, createPerfTimer, createTextDocument, currentSettingsFileVersion, defaultConfigFilenames, defaultFileName, defaultFileName as defaultSettingsFilename, determineFinalDocumentSettings, extractDependencies, extractImportErrors, fileToDocument, fileToTextDocument, finalizeSettings, getCachedFileSize, getDefaultBundledSettingsAsync, getDefaultConfigLoader, getDefaultSettings, getDictionary, getGlobalSettings, getGlobalSettingsAsync, getLanguagesForBasename as getLanguageIdsForBaseFilename, getLanguagesForExt, getLogger, getSources, getSystemFeatureFlags, getVirtualFS, isBinaryFile, isSpellingDictionaryLoadError, loadConfig, loadPnP, mergeInDocSettings, mergeSettings, readRawSettings, readSettings, readSettingsFiles, refreshDictionaryCache, resolveFile, searchForConfig, sectionCSpell, setLogger, shouldCheckDocument, spellCheckDocument, spellCheckFile, suggestionsForWord, suggestionsForWords, traceWords, traceWordsAsync, updateTextDocument, validateText };
export { type CheckTextInfo, type ConfigurationDependencies, type CreateTextDocumentParams, type DetermineFinalDocumentSettingsResult, type Document, DocumentValidator, type DocumentValidatorOptions, ENV_CSPELL_GLOB_ROOT, type ExcludeFilesGlobMap, type ExclusionFunction, exclusionHelper_d as ExclusionHelper, type FeatureFlag, FeatureFlags, ImportError, type ImportFileRefWithError$1 as ImportFileRefWithError, IncludeExcludeFlag, type IncludeExcludeOptions, index_link_d as Link, type Logger, type PerfTimer, type SpellCheckFileOptions, type SpellCheckFileResult, SpellingDictionaryLoadError, type SuggestedWord, SuggestionError, type SuggestionOptions, type SuggestionsForWordResult, text_d as Text, type TextDocument, type TextDocumentLine, type TextDocumentRef, type TextInfoItem, type TraceOptions, type TraceResult, type TraceWordResult, UnknownFeatureFlagError, type ValidationIssue, calcOverrideSettings, checkFilenameMatchesGlob, checkText, checkTextDocument, clearCachedFiles, clearCaches, combineTextAndLanguageSettings, combineTextAndLanguageSettings as constructSettingsForText, createConfigLoader, createPerfTimer, createTextDocument, currentSettingsFileVersion, defaultConfigFilenames, defaultFileName, defaultFileName as defaultSettingsFilename, determineFinalDocumentSettings, extractDependencies, extractImportErrors, fileToDocument, fileToTextDocument, finalizeSettings, getCachedFileSize, getDefaultBundledSettingsAsync, getDefaultConfigLoader, getDefaultSettings, getDictionary, getGlobalSettings, getGlobalSettingsAsync, getLanguagesForBasename as getLanguageIdsForBaseFilename, getLanguagesForExt, getLogger, getSources, getSystemFeatureFlags, getVirtualFS, isBinaryFile, isSpellingDictionaryLoadError, loadConfig, loadPnP, mergeInDocSettings, mergeSettings, readRawSettings, readSettings, readSettingsFiles, refreshDictionaryCache, resolveFile, searchForConfig, sectionCSpell, setLogger, shouldCheckDocument, spellCheckDocument, spellCheckFile, suggestionsForWord, suggestionsForWords, traceWords, traceWordsAsync, updateTextDocument, validateText };
1 change: 1 addition & 0 deletions packages/cspell-lib/fixtures/traceWords/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# Fixtures for traceWords test
14 changes: 14 additions & 0 deletions packages/cspell-lib/fixtures/traceWords/cspell.config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
dictionaryDefinitions:
- name: myDictionary
words: ["café", "bar", "restaurant"]
ignoreWords: ["foo", "bar", "baz"]
suggestWords:
- cafe->café
flagWords:
- baz
dictionaries:
- myDictionary

words: ["foo", "bar", "baz", "cafe"]
flagWords:
- teh # cspell:ignore teh
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ export function getSources(settings: CSpellSettingsWSTO): CSpellSettingsWSTO[] {

type Imports = CSpellSettingsWSTO['__imports'];

function mergeImportRefs(left: CSpellSettingsWSTO, right: CSpellSettingsWSTO = {}): Imports {
function mergeImportRefs(left: CSpellSettingsWSTO, right: CSpellSettingsWSTO = {}): Imports | undefined {
const imports = new Map(left.__imports || []);
if (left.__importRef) {
imports.set(left.__importRef.filename, left.__importRef);
Expand Down
53 changes: 41 additions & 12 deletions packages/cspell-lib/src/lib/SpellingDictionary/Dictionaries.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { CSpellSettings } from '@cspell/cspell-types';
import type { SpellingDictionary, SpellingDictionaryCollection } from 'cspell-dictionary';
import {
createCollection,
Expand Down Expand Up @@ -27,10 +28,21 @@ export async function getDictionaryInternal(settings: CSpellSettingsInternal): P
return _getDictionaryInternal(settings, spellDictionaries);
}

function _getDictionaryInternal(
settings: CSpellSettingsInternal,
spellDictionaries: SpellingDictionary[],
): SpellingDictionaryCollection {
export const specialDictionaryNames = {
words: '[words]',
userWords: '[userWords]',
flagWords: '[flagWords]',
ignoreWords: '[ignoreWords]',
suggestWords: '[suggestWords]',
} as const;

export type DictionaryNameFields = keyof typeof specialDictionaryNames;

export const mapSpecialDictionaryNamesToSettings: Map<string, DictionaryNameFields> = new Map(
Object.entries(specialDictionaryNames).map(([k, v]) => [v, k as DictionaryNameFields] as const),
);

export function getInlineConfigDictionaries(settings: CSpellSettings): SpellingDictionary[] {
const {
words = emptyWords,
userWords = emptyWords,
Expand All @@ -39,34 +51,51 @@ function _getDictionaryInternal(
suggestWords = emptyWords,
} = settings;

const settingsWordsDictionary = createSpellingDictionary(words, '[words]', 'From Settings `words`', {
caseSensitive: true,
weightMap: undefined,
});
const settingsWordsDictionary = createSpellingDictionary(
words,
specialDictionaryNames.words,
'From Settings `words`',
{
caseSensitive: true,
weightMap: undefined,
},
);
const settingsUserWordsDictionary = userWords.length
? createSpellingDictionary(userWords, '[userWords]', 'From Settings `userWords`', {
? createSpellingDictionary(userWords, specialDictionaryNames.userWords, 'From Settings `userWords`', {
caseSensitive: true,
weightMap: undefined,
})
: undefined;
const ignoreWordsDictionary = createIgnoreWordsDictionary(
ignoreWords,
'[ignoreWords]',
specialDictionaryNames.ignoreWords,
'From Settings `ignoreWords`',
);
const flagWordsDictionary = createForbiddenWordsDictionary(flagWords, '[flagWords]', 'From Settings `flagWords`');
const flagWordsDictionary = createForbiddenWordsDictionary(
flagWords,
specialDictionaryNames.flagWords,
'From Settings `flagWords`',
);
const suggestWordsDictionary = createSuggestDictionary(
suggestWords,
'[suggestWords]',
'From Settings `suggestWords`',
);
const dictionaries = [
...spellDictionaries,
settingsWordsDictionary,
settingsUserWordsDictionary,
ignoreWordsDictionary,
flagWordsDictionary,
suggestWordsDictionary,
].filter(isDefined);

return dictionaries;
}

function _getDictionaryInternal(
settings: CSpellSettingsInternal,
spellDictionaries: SpellingDictionary[],
): SpellingDictionaryCollection {
const dictionaries = [...spellDictionaries, ...getInlineConfigDictionaries(settings)];
return createCollection(dictionaries, 'dictionary collection');
}
2 changes: 1 addition & 1 deletion packages/cspell-lib/src/lib/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ export {
export type { SuggestedWord, SuggestionOptions, SuggestionsForWordResult } from './suggestions.js';
export { SuggestionError, suggestionsForWord, suggestionsForWords } from './suggestions.js';
export { DocumentValidator, DocumentValidatorOptions, shouldCheckDocument } from './textValidation/index.js';
export type { TraceOptions, TraceResult } from './trace.js';
export type { TraceOptions, TraceResult, TraceWordResult } from './trace.js';
export { traceWords, traceWordsAsync } from './trace.js';
export { getLogger, Logger, setLogger } from './util/logger.js';
export { resolveFile } from './util/resolveFile.js';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { MappedText, TextOffset as TextOffsetRW } from '@cspell/cspell-type
import type { ExtendedSuggestion } from '../Models/Suggestion.js';
import type { ValidationIssue } from '../Models/ValidationIssue.js';

export type { TextOffset as TextOffsetRW } from '@cspell/cspell-types';
export type TextOffsetRO = Readonly<TextOffsetRW>;

export interface ValidationOptions extends IncludeExcludeOptions {
Expand Down
28 changes: 2 additions & 26 deletions packages/cspell-lib/src/lib/textValidation/docValidator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import { catchPromiseError, toError } from '../util/errors.js';
import { AutoCache } from '../util/simpleCache.js';
import type { MatchRange } from '../util/TextRange.js';
import { uriToFilePath } from '../util/Uri.js';
import { toFilePathOrHref } from '../util/url.js';
import { defaultMaxDuplicateProblems, defaultMaxNumberOfProblems } from './defaultConstants.js';
import { determineTextDocumentSettings } from './determineTextDocumentSettings.js';
import type { TextValidator } from './lineValidatorFactory.js';
Expand All @@ -36,6 +35,7 @@ import type { SimpleRange } from './parsedText.js';
import { createMappedTextSegmenter } from './parsedText.js';
import { settingsToValidateOptions } from './settingsToValidateOptions.js';
import { calcTextInclusionRanges } from './textValidator.js';
import { traceWord } from './traceWord.js';
import type { ValidateTextOptions } from './ValidateTextOptions.js';
import type { MappedTextValidationResult, ValidationOptions } from './ValidationTypes.js';

Expand Down Expand Up @@ -332,27 +332,7 @@ export class DocumentValidator {

public traceWord(word: string) {
assert(this._preparations, ERROR_NOT_PREPARED);
const dictCollection = this._preparations.dictionary;
const config = this._preparations.config;

const opts = {
ignoreCase: true,
allowCompoundWords: config.allowCompoundWords || false,
};

const trace = dictCollection.dictionaries
.map((dict) => ({ dict, findResult: dict.find(word, opts) }))
.map(({ dict, findResult }) => ({
word,
found: !!findResult?.found,
foundWord: findResult?.found || undefined,
forbidden: findResult?.forbidden || false,
noSuggest: findResult?.noSuggest || false,
dictName: dict.name,
dictSource: toFilePathOrHref(dict.source),
errors: normalizeErrors(dict.getErrors?.()),
}));
return trace;
return traceWord(word, this._preparations.dictionary, this._preparations.config);
}

private defaultParser(): Iterable<ParsedText> {
Expand Down Expand Up @@ -556,7 +536,3 @@ function recordPerfTime(timings: PerfTimings, name: string): () => void {
function timePromise<T>(timings: PerfTimings, name: string, p: Promise<T>): Promise<T> {
return p.finally(recordPerfTime(timings, name));
}

function normalizeErrors(errors: Error[] | undefined): Error[] | undefined {
return errors?.length ? errors : undefined;
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import type {
LineValidatorFn,
MappedTextValidationResult,
TextOffsetRO,
TextOffsetRW,
TextValidatorFn,
ValidationIssueRO,
ValidationOptions,
Expand All @@ -26,6 +27,10 @@ interface LineValidator {
dict: CachingDictionary;
}

interface TextOffsetWithLine extends TextOffsetRW {
line?: TextOffsetRO;
}

export function lineValidatorFactory(sDict: SpellingDictionary, options: ValidationOptions): LineValidator {
const {
minWordLength = defaultMinWordLength,
Expand Down Expand Up @@ -147,8 +152,8 @@ export function lineValidatorFactory(sDict: SpellingDictionary, options: Validat
const mismatches: ValidationIssue[] = toArray(
pipe(
Text.extractWordsFromTextOffset(possibleWord),
opFilter(filterAlreadyChecked),
opMap((wo) => ({ ...wo, line: lineSegment.line })),
opFilter((wo: TextOffsetWithLine) => filterAlreadyChecked(wo)),
opMap((wo: TextOffsetWithLine) => ((wo.line = lineSegment.line), wo as ValidationIssue)),
opMap(annotateIsFlagged),
opFilter(rememberFilter((wo) => wo.text.length >= minWordLength || !!wo.isFlagged)),
opConcatMap(checkFullWord),
Expand Down
83 changes: 83 additions & 0 deletions packages/cspell-lib/src/lib/textValidation/traceWord.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import { describe, expect, test } from 'vitest';

import { pathPackageFixturesURL } from '../../test-util/test.locations.cjs';
import type { TextDocumentRef } from '../Models/TextDocument.js';
import { searchForConfig } from '../Settings/index.js';
import { getDictionaryInternal } from '../SpellingDictionary/index.js';
import { toUri } from '../util/Uri.js';
import { determineTextDocumentSettings } from './determineTextDocumentSettings.js';
import type { WordSplits } from './traceWord.js';
import { traceWord } from './traceWord.js';

const fixturesURL = new URL('traceWords/', pathPackageFixturesURL);
const urlReadme = new URL('README.md', fixturesURL);
const expectedConfigURL = new URL('cspell.config.yaml', fixturesURL);

const ac = expect.arrayContaining;
const oc = expect.objectContaining;

describe('traceWord', async () => {
const doc: TextDocumentRef = { uri: toUri(import.meta.url), languageId: 'typescript' };
const fixtureSettings = (await searchForConfig(urlReadme)) || {};
const baseSettings = await determineTextDocumentSettings(doc, fixtureSettings);
const dicts = await getDictionaryInternal(baseSettings);

test('traceWord', () => {
const r = traceWord('trace', dicts, baseSettings);
expect(r.filter((r) => r.found)).toEqual(
ac([
{
word: 'trace',
found: true,
foundWord: 'trace',
forbidden: false,
noSuggest: false,
dictName: 'en_us',
dictSource: expect.any(String),
configSource: undefined,
errors: undefined,
},
]),
);
});

test.each`
word | expected
${'word'} | ${[wft('word')]}
${'word_word'} | ${[wff('word_word'), wft('word'), wft('word')]}
${'word_nword'} | ${[wff('word_nword'), wft('word'), wff('nword')] /* cspell:ignore nword */}
${'ISpellResult'} | ${[wff('ISpellResult'), wft('I'), wft('Spell'), wft('Result')]}
${'ERRORcode'} | ${[wft('ERRORcode'), wft('ERROR'), wft('code')]}
`('traceWord splits $word', ({ word, expected }) => {
const r = traceWord(word, dicts, baseSettings);
expect(r.splits).toEqual(expected);
});

test.each`
word | expected
${'word_word'} | ${{ ...wft('word'), dictName: 'en_us' }}
${'ISpellResult'} | ${{ ...wft('Result'), foundWord: 'result', dictName: 'en_us' }}
${'ERRORcode'} | ${{ ...wft('ERRORcode'), foundWord: 'errorcode', dictName: 'node' }}
${'ERRORcode'} | ${{ ...wft('ERROR'), foundWord: 'error', dictName: 'en_us' }}
${'apple-pie'} | ${{ ...wft('pie'), dictName: 'en_us' }}
${"can't"} | ${{ ...wft("can't"), dictName: 'en_us' }}
${'canNOT'} | ${{ ...wft('canNOT'), foundWord: 'cannot', dictName: 'en_us' }}
${'baz'} | ${{ ...wft('baz'), foundWord: 'baz', dictName: '[words]', dictSource: expectedConfigURL.href }}
`('traceWord check found $word', ({ word, expected }) => {
const r = traceWord(word, dicts, baseSettings);
const matching = r.filter((r) => r.word === expected.word && r.found === expected.found);
expect(matching).toEqual(ac([oc(expected)]));
});
});

function wf(word: string, found: boolean): WordSplits {
return { word, found };
}

function wft(word: string): WordSplits {
return wf(word, true);
}

function wff(word: string): WordSplits {
return wf(word, false);
}
Loading

0 comments on commit 1c53140

Please sign in to comment.