Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adopts "containsBalancedBrackets" flag from vscode/text-mate. #146962

Merged
merged 5 commits into from
Apr 12, 2022
Merged

Adopts "containsBalancedBrackets" flag from vscode/text-mate. #146962

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9aaa850
Adopts "containsBalancedBrackets" flag from vscode/text-mate.
hediet Apr 6, 2022
37baf56
Merge remote-tracking branch 'origin/main' into hediet/containsBalanc…
alexdima Apr 7, 2022
91d0063
Fix bad merge
alexdima Apr 7, 2022
cfb8ec1
Merge branch 'main' into hediet/containsBalancedBrackets
hediet Apr 11, 2022
2b14c47
Fixes test failures.
hediet Apr 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion extensions/shellscript/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,13 @@
{
"language": "shellscript",
"scopeName": "source.shell",
"path": "./syntaxes/shell-unix-bash.tmLanguage.json"
"path": "./syntaxes/shell-unix-bash.tmLanguage.json",
"balancedBracketScopes": [
"*"
],
"unbalancedBracketScopes": [
"meta.scope.case-pattern.shell"
]
}
],
"configurationDefaults": {
Expand Down
3 changes: 2 additions & 1 deletion src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,11 @@ class NonPeekableTextBufferTokenizer {
}

const isOther = TokenMetadata.getTokenType(tokenMetadata) === StandardTokenType.Other;
const containsBracketType = TokenMetadata.containsBalancedBrackets(tokenMetadata);

const endOffset = lineTokens.getEndOffset(this.lineTokenOffset);
// Is there a bracket token next? Only consume text.
if (isOther && endOffset !== this.lineCharOffset) {
if (containsBracketType && isOther && endOffset !== this.lineCharOffset) {
const languageId = lineTokens.getLanguageId(this.lineTokenOffset);
const text = this.line.substring(this.lineCharOffset, endOffset);

Expand Down
2 changes: 2 additions & 0 deletions src/vs/editor/common/tokens/contiguousTokensStore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,5 +211,7 @@ function getDefaultMetadata(topLevelLanguageId: LanguageId): number {
| (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET)
| (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET)
| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
// If there is no grammar, we just take a guess and try to match brackets.
| (MetadataConsts.BALANCED_BRACKETS_MASK)
) >>> 0;
}
21 changes: 14 additions & 7 deletions src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ suite('Bracket Pair Colorizer - Tokenizer', () => {

const denseKeyProvider = new DenseKeyProvider<string>();

const tStandard = (text: string) => new TokenInfo(text, encodedMode1, StandardTokenType.Other);
const tComment = (text: string) => new TokenInfo(text, encodedMode1, StandardTokenType.Comment);
const tStandard = (text: string) => new TokenInfo(text, encodedMode1, StandardTokenType.Other, true);
const tComment = (text: string) => new TokenInfo(text, encodedMode1, StandardTokenType.Comment, true);
const document = new TokenizedDocument([
tStandard(' { } '), tStandard('be'), tStandard('gin end'), tStandard('\n'),
tStandard('hello'), tComment('{'), tStandard('}'),
Expand Down Expand Up @@ -189,16 +189,23 @@ class TokenizedDocument {
}

class TokenInfo {
constructor(public readonly text: string, public readonly languageId: LanguageId, public readonly tokenType: StandardTokenType) { }
constructor(
public readonly text: string,
public readonly languageId: LanguageId,
public readonly tokenType: StandardTokenType,
public readonly hasBalancedBrackets: boolean,
) { }

getMetadata(): number {
return (
(this.languageId << MetadataConsts.LANGUAGEID_OFFSET)
| (this.tokenType << MetadataConsts.TOKEN_TYPE_OFFSET)
) >>> 0;
(((this.languageId << MetadataConsts.LANGUAGEID_OFFSET) |
(this.tokenType << MetadataConsts.TOKEN_TYPE_OFFSET)) >>>
0) |
(this.hasBalancedBrackets ? MetadataConsts.BALANCED_BRACKETS_MASK : 0)
);
}

withText(text: string): TokenInfo {
return new TokenInfo(text, this.languageId, this.tokenType);
return new TokenInfo(text, this.languageId, this.tokenType, this.hasBalancedBrackets);
}
}
9 changes: 7 additions & 2 deletions src/vs/editor/test/common/model/textModelWithTokens.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ suite('TextModelWithTokens - bracket matching', () => {
});
});

suite('TextModelWithTokens', () => {
suite('TextModelWithTokens 2', () => {

test('bracket matching 3', () => {
const text = [
Expand Down Expand Up @@ -359,10 +359,12 @@ suite('TextModelWithTokens', () => {
const otherMetadata1 = (
(encodedMode1 << MetadataConsts.LANGUAGEID_OFFSET)
| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
| (MetadataConsts.BALANCED_BRACKETS_MASK)
) >>> 0;
const otherMetadata2 = (
(encodedMode2 << MetadataConsts.LANGUAGEID_OFFSET)
| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
| (MetadataConsts.BALANCED_BRACKETS_MASK)
) >>> 0;

const tokenizationSupport: ITokenizationSupport = {
Expand Down Expand Up @@ -441,7 +443,10 @@ suite('TextModelWithTokens', () => {
model.tokenization.forceTokenization(2);
model.tokenization.forceTokenization(3);

assert.deepStrictEqual(model.bracketPairs.matchBracket(new Position(2, 14)), [new Range(2, 13, 2, 14), new Range(2, 18, 2, 19)]);
assert.deepStrictEqual(
model.bracketPairs.matchBracket(new Position(2, 14)),
[new Range(2, 13, 2, 14), new Range(2, 18, 2, 19)]
);

disposables.dispose();
});
Expand Down
32 changes: 16 additions & 16 deletions src/vs/editor/test/common/model/tokensStore.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,22 +194,22 @@ suite('TokensStore', () => {
}

assert.deepStrictEqual(decodedTokens, [
20, 0b10000000001000000000000001,
24, 0b10000001111000000000000001,
25, 0b10000000001000000000000001,
27, 0b10000001111000000000000001,
28, 0b10000000001000000000000001,
29, 0b10000000001000000000000001,
31, 0b10000010000000000000000001,
32, 0b10000000001000000000000001,
33, 0b10000000001000000000000001,
34, 0b10000000001000000000000001,
36, 0b10000000110000000000000001,
37, 0b10000000001000000000000001,
38, 0b10000000001000000000000001,
42, 0b10000001111000000000000001,
43, 0b10000000001000000000000001,
47, 0b10000001011000000000000001
20, 0b10000000001000010000000001,
24, 0b10000001111000010000000001,
25, 0b10000000001000010000000001,
27, 0b10000001111000010000000001,
28, 0b10000000001000010000000001,
29, 0b10000000001000010000000001,
31, 0b10000010000000010000000001,
32, 0b10000000001000010000000001,
33, 0b10000000001000010000000001,
34, 0b10000000001000010000000001,
36, 0b10000000110000010000000001,
37, 0b10000000001000010000000001,
38, 0b10000000001000010000000001,
42, 0b10000001111000010000000001,
43, 0b10000000001000010000000001,
47, 0b10000001011000010000000001
]);

model.dispose();
Expand Down
12 changes: 12 additions & 0 deletions src/vs/workbench/services/textMate/browser/abstractTextMateService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,13 +133,25 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
validLanguageId = grammar.language;
}

function asStringArray(array: unknown, defaultValue: string[]): string[] {
if (!Array.isArray(array)) {
return defaultValue;
}
if (!array.every(e => typeof e === 'string')) {
return defaultValue;
}
return array;
}

this._grammarDefinitions.push({
location: grammarLocation,
language: validLanguageId ? validLanguageId : undefined,
scopeName: grammar.scopeName,
embeddedLanguages: embeddedLanguages,
tokenTypes: tokenTypes,
injectTo: grammar.injectTo,
balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ['*']),
unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
});

if (validLanguageId) {
Expand Down
4 changes: 4 additions & 0 deletions src/vs/workbench/services/textMate/browser/textMateWorker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ export interface IValidGrammarDefinitionDTO {
embeddedLanguages: IValidEmbeddedLanguagesMap;
tokenTypes: IValidTokenTypeMap;
injectTo?: string[];
balancedBracketSelectors: string[];
unbalancedBracketSelectors: string[];
}

export interface ICreateData {
Expand Down Expand Up @@ -143,6 +145,8 @@ export class TextMateWorker {
embeddedLanguages: def.embeddedLanguages,
tokenTypes: def.tokenTypes,
injectTo: def.injectTo,
balancedBracketSelectors: def.balancedBracketSelectors,
unbalancedBracketSelectors: def.unbalancedBracketSelectors,
};
});
this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions);
Expand Down
11 changes: 10 additions & 1 deletion src/vs/workbench/services/textMate/common/TMGrammarFactory.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,16 @@ export class TMGrammarFactory extends Disposable {
let grammar: IGrammar | null;

try {
grammar = await this._grammarRegistry.loadGrammarWithConfiguration(scopeName, encodedLanguageId, { embeddedLanguages, tokenTypes: <any>grammarDefinition.tokenTypes });
grammar = await this._grammarRegistry.loadGrammarWithConfiguration(
scopeName,
encodedLanguageId,
{
embeddedLanguages,
tokenTypes: <any>grammarDefinition.tokenTypes,
balancedBracketSelectors: grammarDefinition.balancedBracketSelectors,
unbalancedBracketSelectors: grammarDefinition.unbalancedBracketSelectors,
}
);
} catch (err) {
if (err.message && err.message.startsWith('No grammar provided for')) {
// No TM grammar defined
Expand Down
20 changes: 19 additions & 1 deletion src/vs/workbench/services/textMate/common/TMGrammars.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ export interface ITMSyntaxExtensionPoint {
embeddedLanguages: IEmbeddedLanguagesMap;
tokenTypes: TokenTypesContribution;
injectTo: string[];
balancedBracketScopes: string[];
unbalancedBracketScopes: string[];
}

export const grammarsExtPoint: IExtensionPoint<ITMSyntaxExtensionPoint[]> = ExtensionsRegistry.registerExtensionPoint<ITMSyntaxExtensionPoint[]>({
Expand Down Expand Up @@ -64,7 +66,23 @@ export const grammarsExtPoint: IExtensionPoint<ITMSyntaxExtensionPoint[]> = Exte
items: {
type: 'string'
}
}
},
balancedBracketScopes: {
description: nls.localize('vscode.extension.contributes.grammars.balancedBracketScopes', 'Defines which scope names contain balanced brackets.'),
type: 'array',
items: {
type: 'string'
},
default: ['*'],
},
unbalancedBracketScopes: {
description: nls.localize('vscode.extension.contributes.grammars.unbalancedBracketScopes', 'Defines which scope names do not contain balanced brackets.'),
type: 'object',
items: {
type: 'string'
},
default: [],
},
},
required: ['scopeName', 'path']
}
Expand Down
2 changes: 2 additions & 0 deletions src/vs/workbench/services/textMate/common/TMScopeRegistry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ export interface IValidGrammarDefinition {
embeddedLanguages: IValidEmbeddedLanguagesMap;
tokenTypes: IValidTokenTypeMap;
injectTo?: string[];
balancedBracketSelectors: string[];
unbalancedBracketSelectors: string[];
}

export interface IValidTokenTypeMap {
Expand Down