From 77add4162bbe4ea670d2b8f1fca3097d54ca193d Mon Sep 17 00:00:00 2001 From: Pankaj Khandelwal Date: Mon, 21 Sep 2020 10:40:48 +0200 Subject: [PATCH 1/2] Fixes microsoft/monaco-editor#1999: Use parent stack to keep track of the current parent and previous parents for any particular key or value part --- src/tokenization.ts | 99 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 80 insertions(+), 19 deletions(-) diff --git a/src/tokenization.ts b/src/tokenization.ts index ab8e3e0..758e0ca 100644 --- a/src/tokenization.ts +++ b/src/tokenization.ts @@ -10,7 +10,7 @@ export function createTokenizationSupport( supportComments: boolean ): languages.TokensProvider { return { - getInitialState: () => new JSONState(null, null), + getInitialState: () => new JSONState(null, null, false, []), tokenize: (line, state, offsetDelta?, stopAtOffset?) => tokenize( supportComments, @@ -34,18 +34,60 @@ export const TOKEN_PROPERTY_NAME = 'string.key.json'; export const TOKEN_COMMENT_BLOCK = 'comment.block.json'; export const TOKEN_COMMENT_LINE = 'comment.line.json'; +enum JSONParent { + Object = 0, + Array = 1 +} + class JSONState implements languages.IState { private _state: languages.IState; public scanError: json.ScanError; - - constructor(state: languages.IState, scanError: json.ScanError) { + public lastWasColon: boolean; + public parents: JSONParent[]; + + constructor( + state: languages.IState, + scanError: json.ScanError, + lastWasColon: boolean, + parents: JSONParent[] + ) { this._state = state; this.scanError = scanError; + this.lastWasColon = lastWasColon; + this.parents = parents; + } + + private static areArraysEqual(first: JSONParent[], second: JSONParent[]) { + if (first === second) { + return true; + } + + if (first == null || second === null) { + return false; + } + + if (first.length !== second.length) { + return false; + } + + let index = -1; + while (++index < first.length) { + if (first[index] !== second[index]) { + return false; + } + } + + return true; } public clone(): JSONState { - return new JSONState(this._state, this.scanError); + return new JSONState( + this._state, + this.scanError, + this.lastWasColon, + this.parents + ); } public equals(other: languages.IState): boolean { @@ -55,7 +97,11 @@ class JSONState implements languages.IState { if (!other || !(other instanceof JSONState)) { return false; } - return this.scanError === (other).scanError; + return ( + this.scanError === (other).scanError && + this.lastWasColon === (other).lastWasColon && + JSONState.areArraysEqual(this.parents, (other).parents) + ); } public getStateData(): languages.IState { @@ -89,7 +135,9 @@ function tokenize( break; } - const scanner = json.createScanner(line); + let scanner = json.createScanner(line), + lastWasColon = state.lastWasColon, + parents = Array.from(state.parents); const ret: languages.ILineTokens = { tokens: [], @@ -123,46 +171,54 @@ function tokenize( // brackets and type switch (kind) { case json.SyntaxKind.OpenBraceToken: + parents.push(JSONParent.Object); type = TOKEN_DELIM_OBJECT; + lastWasColon = false; break; case json.SyntaxKind.CloseBraceToken: + parents.pop(); type = TOKEN_DELIM_OBJECT; + lastWasColon = false; break; case json.SyntaxKind.OpenBracketToken: + parents.push(JSONParent.Array); type = TOKEN_DELIM_ARRAY; + lastWasColon = false; break; case json.SyntaxKind.CloseBracketToken: + parents.pop(); type = TOKEN_DELIM_ARRAY; + lastWasColon = false; break; case json.SyntaxKind.ColonToken: - for (let i = ret.tokens.length - 1; i >= 0; i--) { - const token = ret.tokens[i]; - if (token.scopes === '' || token.scopes === TOKEN_COMMENT_BLOCK) { - continue; - } - if (token.scopes === TOKEN_VALUE_STRING) { - // !change previous token to property name! - token.scopes = TOKEN_PROPERTY_NAME; - } - break; - } type = TOKEN_DELIM_COLON; + lastWasColon = true; break; case json.SyntaxKind.CommaToken: type = TOKEN_DELIM_COMMA; + lastWasColon = false; break; case json.SyntaxKind.TrueKeyword: case json.SyntaxKind.FalseKeyword: type = TOKEN_VALUE_BOOLEAN; + lastWasColon = false; break; case json.SyntaxKind.NullKeyword: type = TOKEN_VALUE_NULL; + lastWasColon = false; break; case json.SyntaxKind.StringLiteral: - type = TOKEN_VALUE_STRING; + let currentParent = parents.length + ? parents[parents.length - 1] + : JSONParent.Object; + let inArray = currentParent === JSONParent.Array; + type = + lastWasColon || inArray ? TOKEN_VALUE_STRING : TOKEN_PROPERTY_NAME; + lastWasColon = false; break; case json.SyntaxKind.NumericLiteral: type = TOKEN_VALUE_NUMBER; + lastWasColon = false; break; } @@ -178,7 +234,12 @@ function tokenize( } } - ret.endState = new JSONState(state.getStateData(), scanner.getTokenError()); + ret.endState = new JSONState( + state.getStateData(), + scanner.getTokenError(), + lastWasColon, + parents + ); ret.tokens.push({ startIndex: offset, scopes: type From bb6231ad91b33e41462ad563a6245680a993b42e Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 21 Sep 2020 10:56:39 +0200 Subject: [PATCH 2/2] Replace array with immutable stack --- src/tokenization.ts | 101 ++++++++++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/src/tokenization.ts b/src/tokenization.ts index 758e0ca..73a4010 100644 --- a/src/tokenization.ts +++ b/src/tokenization.ts @@ -10,7 +10,7 @@ export function createTokenizationSupport( supportComments: boolean ): languages.TokensProvider { return { - getInitialState: () => new JSONState(null, null, false, []), + getInitialState: () => new JSONState(null, null, false, null), tokenize: (line, state, offsetDelta?, stopAtOffset?) => tokenize( supportComments, @@ -34,23 +34,67 @@ export const TOKEN_PROPERTY_NAME = 'string.key.json'; export const TOKEN_COMMENT_BLOCK = 'comment.block.json'; export const TOKEN_COMMENT_LINE = 'comment.line.json'; -enum JSONParent { +const enum JSONParent { Object = 0, Array = 1 } +class ParentsStack { + constructor( + public readonly parent: ParentsStack | null, + public readonly type: JSONParent + ) {} + + public static pop(parents: ParentsStack | null): ParentsStack | null { + if (parents) { + return parents.parent; + } + return null; + } + + public static push( + parents: ParentsStack | null, + type: JSONParent + ): ParentsStack { + return new ParentsStack(parents, type); + } + + public static equals( + a: ParentsStack | null, + b: ParentsStack | null + ): boolean { + if (!a && !b) { + return true; + } + if (!a || !b) { + return false; + } + while (a && b) { + if (a === b) { + return true; + } + if (a.type !== b.type) { + return false; + } + a = a.parent; + b = b.parent; + } + return true; + } +} + class JSONState implements languages.IState { private _state: languages.IState; public scanError: json.ScanError; public lastWasColon: boolean; - public parents: JSONParent[]; + public parents: ParentsStack | null; constructor( state: languages.IState, scanError: json.ScanError, lastWasColon: boolean, - parents: JSONParent[] + parents: ParentsStack | null ) { this._state = state; this.scanError = scanError; @@ -58,29 +102,6 @@ class JSONState implements languages.IState { this.parents = parents; } - private static areArraysEqual(first: JSONParent[], second: JSONParent[]) { - if (first === second) { - return true; - } - - if (first == null || second === null) { - return false; - } - - if (first.length !== second.length) { - return false; - } - - let index = -1; - while (++index < first.length) { - if (first[index] !== second[index]) { - return false; - } - } - - return true; - } - public clone(): JSONState { return new JSONState( this._state, @@ -98,9 +119,9 @@ class JSONState implements languages.IState { return false; } return ( - this.scanError === (other).scanError && - this.lastWasColon === (other).lastWasColon && - JSONState.areArraysEqual(this.parents, (other).parents) + this.scanError === other.scanError && + this.lastWasColon === other.lastWasColon && + ParentsStack.equals(this.parents, other.parents) ); } @@ -135,9 +156,9 @@ function tokenize( break; } - let scanner = json.createScanner(line), - lastWasColon = state.lastWasColon, - parents = Array.from(state.parents); + const scanner = json.createScanner(line); + let lastWasColon = state.lastWasColon; + let parents = state.parents; const ret: languages.ILineTokens = { tokens: [], @@ -171,22 +192,22 @@ function tokenize( // brackets and type switch (kind) { case json.SyntaxKind.OpenBraceToken: - parents.push(JSONParent.Object); + parents = ParentsStack.push(parents, JSONParent.Object); type = TOKEN_DELIM_OBJECT; lastWasColon = false; break; case json.SyntaxKind.CloseBraceToken: - parents.pop(); + parents = ParentsStack.pop(parents); type = TOKEN_DELIM_OBJECT; lastWasColon = false; break; case json.SyntaxKind.OpenBracketToken: - parents.push(JSONParent.Array); + parents = ParentsStack.push(parents, JSONParent.Array); type = TOKEN_DELIM_ARRAY; lastWasColon = false; break; case json.SyntaxKind.CloseBracketToken: - parents.pop(); + parents = ParentsStack.pop(parents); type = TOKEN_DELIM_ARRAY; lastWasColon = false; break; @@ -208,10 +229,8 @@ function tokenize( lastWasColon = false; break; case json.SyntaxKind.StringLiteral: - let currentParent = parents.length - ? parents[parents.length - 1] - : JSONParent.Object; - let inArray = currentParent === JSONParent.Array; + const currentParent = parents ? parents.type : JSONParent.Object; + const inArray = currentParent === JSONParent.Array; type = lastWasColon || inArray ? TOKEN_VALUE_STRING : TOKEN_PROPERTY_NAME; lastWasColon = false;