Skip to content

Commit

Permalink
Use Cst tree's original implementation for the reassembling step
Browse files Browse the repository at this point in the history
  • Loading branch information
@Lotes
Lotes committed Aug 14, 2024
1 parent 94bab42 commit d4476b0
Show file tree
Hide file tree
Showing 4 changed files with 168 additions and 213 deletions.
75 changes: 51 additions & 24 deletions packages/langium/src/serializer/reassembler/AstAssemblerInstruction.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
* terms of the MIT License, which is available in the project root.
******************************************************************************/

import type { Range } from 'vscode-languageserver-protocol';
import type { Grammar, AbstractElement} from '../../languages/generated/ast.js';
import { isAbstractElement } from '../../languages/generated/ast.js';
import { streamAst } from '../../utils/ast-utils.js';
Expand All @@ -14,14 +15,19 @@ export enum InstructionType {
Element,
TokenType,
Property,
Properties,
PropertyArray,
LinkNode,
LinkNodes,
LinkNodeArray,
Reference,
References,
ReferenceArray,
Empty,
Error,
Return
Return,

RootCstNode,
CompositeCstNode,
LeafCstNode,
PopCstNode
}
export enum NodeType {
Cst,
Expand All @@ -46,66 +52,57 @@ export namespace Instructions {
}
export interface TokenType extends AstAssemblerInstructionBase {
$type: InstructionType.TokenType;
sourceKind: NodeType;
sourceId: number;
property: string;
tokenName: string;
}
export interface Element extends AstAssemblerInstructionBase {
$type: InstructionType.Element;
sourceKind: NodeType;
sourceId: number;
property: string;
value: number;
}
export interface Property extends AstAssemblerInstructionBase {
$type: InstructionType.Property;
sourceKind: NodeType;
sourceId: number;
property: string;
value: number;
value: number | boolean | string | bigint;
}
export interface Properties extends AstAssemblerInstructionBase {
$type: InstructionType.Properties;
sourceKind: NodeType;
export interface PropertyArray extends AstAssemblerInstructionBase {
$type: InstructionType.PropertyArray;
sourceId: number;
property: string;
values: Array<number | boolean | string | bigint>;
}
export interface Reference extends AstAssemblerInstructionBase {
$type: InstructionType.Reference;
sourceKind: NodeType;
sourceId: number;
property: string;
refText: string;
refNode?: number;
}
export interface References extends AstAssemblerInstructionBase {
$type: InstructionType.References;
sourceKind: NodeType;
export interface ReferenceArray extends AstAssemblerInstructionBase {
$type: InstructionType.ReferenceArray;
sourceId: number;
property: string;
references: ReferenceData[];
}
export interface LinkNode extends AstAssemblerInstructionBase {
$type: InstructionType.LinkNode;
sourceKind: NodeType;
sourceId: number;
targetKind: NodeType;
property: string;
targetId: number;
}
export interface LinkNodes extends AstAssemblerInstructionBase {
$type: InstructionType.LinkNodes;
sourceKind: NodeType;
export interface LinkNodeArray extends AstAssemblerInstructionBase {
$type: InstructionType.LinkNodeArray;
sourceId: number;
targetKind: NodeType;
property: string;
targetIds: number[];
}
export interface Empty extends AstAssemblerInstructionBase {
$type: InstructionType.Empty;
sourceKind: NodeType;
sourceId: number;
property: string;
}
Expand All @@ -118,20 +115,50 @@ export namespace Instructions {
$type: InstructionType.Return;
rootAstNodeId: number;
}

export interface RootCstNode extends AstAssemblerInstructionBase {
$type: InstructionType.RootCstNode;
input: string;
astNodeId: number|undefined;
}
export interface CompositeCstNode extends AstAssemblerInstructionBase {
$type: InstructionType.CompositeCstNode;
elementId: number;
astNodeId: number|undefined;
}
export interface LeafCstNode extends AstAssemblerInstructionBase {
$type: InstructionType.LeafCstNode;
tokenOffset: number;
tokenLength: number;
tokenTypeName: string;
elementId: number;
hidden: boolean;
range: Range;
astNodeId: number|undefined;
}
export interface PopCstNode extends AstAssemblerInstructionBase {
$type: InstructionType.PopCstNode;
}
}

export type AstAssemblerInstruction = Instructions.Allocate
| Instructions.TokenType
| Instructions.Property
| Instructions.Element
| Instructions.Properties
| Instructions.PropertyArray
| Instructions.Reference
| Instructions.References
| Instructions.ReferenceArray
| Instructions.LinkNode
| Instructions.LinkNodes
| Instructions.LinkNodeArray
| Instructions.Empty
| Instructions.Error
| Instructions.Return;
| Instructions.Return

| Instructions.RootCstNode
| Instructions.CompositeCstNode
| Instructions.LeafCstNode
| Instructions.PopCstNode
;

export function createGrammarElementIdMap(grammar: Grammar) {
const result = new BiMap<AbstractElement, number>();
Expand Down
150 changes: 39 additions & 111 deletions packages/langium/src/serializer/reassembler/AstDisassembler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,117 +41,52 @@ export class DefaultAstDisassembler implements AstDisassembler {
};

//send cst nodes
let cstNodeStack: Array<Mutable<CstNode>> = [];
for (const node of streamCst(cstRoot)) {
assertType<Mutable<CstNode>>(node);
const sourceKind = NodeType.Cst;
const sourceId = this.cstNodeToId.get(node)!;

const setTokenType = (property: string, tokenName: string) => {
const instr = <Instructions.TokenType>{
$type: InstructionType.TokenType,
sourceKind,
sourceId,
property,
tokenName
};
if (this.deleteOriginal) {
assertType<Record<string, undefined>>(node);
node[property] = undefined!;
}
return instr;
};

const setProperty = (property: string, value: number | boolean | string | bigint) => {
const instr = <Instructions.Property>{
$type: InstructionType.Property,
sourceKind,
sourceId,
property,
value
};
if (this.deleteOriginal) {
assertType<Record<string, undefined>>(node);
node[property] = undefined!;
}
return instr;
};

const setLink = (property: string, type: NodeType, index: number) => {
const instr = <Instructions.LinkNode>{
$type: InstructionType.LinkNode,
sourceKind,
sourceId,
targetKind: type,
targetId: index,
property,
if (isRootCstNode(node)) {
yield <Instructions.RootCstNode>{
$type: InstructionType.RootCstNode,
input: node.fullText,
astNodeId: this.astNodeToId.get(node.astNode)
};
if (this.deleteOriginal) {
assertType<Record<string, undefined>>(node);
node[property] = undefined!;
cstNodeStack = [node];
} else if(isCompositeCstNode(node)) {
while(cstNodeStack[cstNodeStack.length-1] !== node.container) {
cstNodeStack.pop();
yield <Instructions.PopCstNode>{
$type: InstructionType.PopCstNode
};
}
return instr;
};

const setLinks = (property: string, type: NodeType, indices: number[]) => {
const instr = <Instructions.LinkNodes>{
$type: InstructionType.LinkNodes,
sourceKind,
sourceId,
targetKind: type,
targetIds: indices,
property,
yield <Instructions.CompositeCstNode>{
$type: InstructionType.CompositeCstNode,
elementId: this.grammarElementIdMap.get(node.grammarSource)!,
astNodeId: this.astNodeToId.get(node.astNode)
};
if (this.deleteOriginal) {
assertType<Record<string, undefined>>(node);
node[property] = undefined!;
cstNodeStack.push(node);
} else if(isLeafCstNode(node)) {
while(cstNodeStack[cstNodeStack.length-1] !== node.container) {
cstNodeStack.pop();
yield <Instructions.PopCstNode>{
$type: InstructionType.PopCstNode
};
}
return instr;
};

const setElement = (property: string, value: number) => {
const instr = <Instructions.Element>{
$type: InstructionType.Element,
sourceKind,
sourceId,
property,
value
yield <Instructions.LeafCstNode>{
$type: InstructionType.LeafCstNode,
elementId: this.grammarElementIdMap.get(node.grammarSource)!,
hidden: node.hidden,
range: node.range,
tokenTypeName: node.tokenType.name,
tokenOffset: node.offset,
tokenLength: node.length,
astNodeId: this.astNodeToId.get(node.astNode)
};
if (this.deleteOriginal) {
assertType<Record<string, undefined>>(node);
node[property] = undefined!;
}
return instr;
};

if (isRootCstNode(node)) {
yield setProperty('fullText', node.fullText);
} else {
// Note: This returns undefined for hidden nodes (i.e. comments)
yield setElement('grammarSource', this.grammarElementIdMap.get(node.grammarSource)!);
}
yield setProperty('hidden', node.hidden);
yield setLink('astNode', NodeType.Ast, this.astNodeToId.get(node.astNode)!);
if (isCompositeCstNode(node)) {
yield setLinks('content', NodeType.Cst, node.content.map(c => this.cstNodeToId.get(c)!));
} else if (isLeafCstNode(node)) {
yield setTokenType('tokenType', node.tokenType.name);
}
yield setProperty('offset', node.offset);
yield setProperty('length', node.length);
yield setLink('root', NodeType.Cst, this.cstNodeToId.get(node.root)!);
if (node.container) {
yield setLink('container', NodeType.Cst, this.cstNodeToId.get(node.container)!);
}
yield setProperty('startLine', node.range.start.line);
yield setProperty('startColumn', node.range.start.character);
yield setProperty('endLine', node.range.end.line);
yield setProperty('endColumn', node.range.end.character);
}

//send ast nodes
for (const node of streamAst(astNode)) {
assertType<Mutable<AstNode>>(node);
const sourceKind = NodeType.Ast;
const sourceId = this.astNodeToId.get(node)!;

const cleanUp = (property: string) => {
Expand All @@ -164,7 +99,6 @@ export class DefaultAstDisassembler implements AstDisassembler {
const setProperty = (property: string, value: number | boolean | string | bigint) => {
const instr = <Instructions.Property>{
$type: InstructionType.Property,
sourceKind,
sourceId,
property,
value
Expand All @@ -177,9 +111,8 @@ export class DefaultAstDisassembler implements AstDisassembler {
};

const setProperties = (property: string, values: Array<number | boolean | string | bigint>) => {
const instr = <Instructions.Properties>{
$type: InstructionType.Properties,
sourceKind,
const instr = <Instructions.PropertyArray>{
$type: InstructionType.PropertyArray,
sourceId,
property,
values
Expand All @@ -191,7 +124,6 @@ export class DefaultAstDisassembler implements AstDisassembler {
const setLink = (property: string, type: NodeType, index: number) => {
const instr = <Instructions.LinkNode>{
$type: InstructionType.LinkNode,
sourceKind,
sourceId,
targetKind: type,
targetId: index,
Expand All @@ -205,9 +137,8 @@ export class DefaultAstDisassembler implements AstDisassembler {
};

const setLinks = (property: string, type: NodeType, indices: number[]) => {
const instr = <Instructions.LinkNodes>{
$type: InstructionType.LinkNodes,
sourceKind,
const instr = <Instructions.LinkNodeArray>{
$type: InstructionType.LinkNodeArray,
sourceId,
targetKind: type,
targetIds: indices,
Expand All @@ -218,9 +149,8 @@ export class DefaultAstDisassembler implements AstDisassembler {
};

const setReferences = (property: string, references: ReferenceData[]) => {
const instr = <Instructions.References>{
$type: InstructionType.References,
sourceKind,
const instr = <Instructions.ReferenceArray>{
$type: InstructionType.ReferenceArray,
sourceId,
property,
references
Expand All @@ -232,7 +162,6 @@ export class DefaultAstDisassembler implements AstDisassembler {
const setReference = (property: string, reference: ReferenceData) => {
const instr = <Instructions.Reference>{
$type: InstructionType.Reference,
sourceKind,
sourceId,
property,
...reference
Expand All @@ -244,7 +173,6 @@ export class DefaultAstDisassembler implements AstDisassembler {
const setEmpty = (property: string) => {
const instr = <Instructions.Empty>{
$type: InstructionType.Empty,
sourceKind,
sourceId,
property,
};
Expand Down
Loading

0 comments on commit d4476b0

Please sign in to comment.