From 90583da280eb93d6557812672c5d352d2a40d601 Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Fri, 20 Dec 2019 14:22:03 +0300 Subject: [PATCH 1/9] Change grammar Add Tokenizer support to work properly with InputStreams. Signed-off-by: Evgeniy --- .../bsl/parser/JMXBSLLexerTest.java | 91 ++- src/main/antlr/BSLLexer.g4 | 2 +- src/main/antlr/BSLParser.g4 | 530 +++++++++--------- .../bsl/parser/BSLExtendedParser.java | 78 +-- .../_1c_syntax/bsl/parser/Tokenizer.java | 47 +- .../_1c_syntax/bsl/parser/BSLParserTest.java | 149 +++-- .../_1c_syntax/bsl/parser/TokenizerTest.java | 5 +- 7 files changed, 467 insertions(+), 435 deletions(-) diff --git a/src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java b/src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java index 3db39d41..12dd0924 100644 --- a/src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java +++ b/src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java @@ -24,41 +24,80 @@ import org.antlr.v4.runtime.Lexer; import org.apache.commons.io.IOUtils; import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.profile.StackProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.lang.reflect.InvocationTargetException; -import java.nio.charset.StandardCharsets; +import java.nio.Buffer; +import java.nio.charset.Charset; +import java.util.concurrent.TimeUnit; -@BenchmarkMode(Mode.SampleTime) -@Warmup(iterations = 2) // число итераций для прогрева нашей функции -@Measurement(iterations = 2, batchSize = 2) -@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 3) +@Measurement(iterations = 5) +@OutputTimeUnit(TimeUnit.MILLISECONDS) public class JMXBSLLexerTest { - @Param({"BSLLexer"}) - public String lexerClassName; - private String content; + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .addProfiler(StackProfiler.class) + .addProfiler(GCProfiler.class) + .build(); - public JMXBSLLexerTest() { - final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - try (InputStream inputStream = classLoader.getResourceAsStream("Module.bsl")) { - assert inputStream != null; - content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - } catch (IOException e) { - e.printStackTrace(); + new Runner(opt).run(); } - } - @Benchmark - public void testCharStream() - throws ClassNotFoundException, InvocationTargetException, InstantiationException, IllegalAccessException { + @Benchmark + @Fork(1) + public void testLexer(Blackhole blackhole, MyState state) throws IOException { + Tokenizer tokenizer; + if (state.mode.equals("As stream")) { + tokenizer = new Tokenizer(Thread.currentThread().getContextClassLoader().getResourceAsStream(state.fileName), state.lexer); + } else { + tokenizer = new Tokenizer(state.content, state.lexer); + } + blackhole.consume(tokenizer.getTokens()); + } + + @Benchmark + @Fork(1) + public void testParser(Blackhole blackhole, MyState state) throws IOException { + Tokenizer tokenizer; + if (state.mode.equals("As stream")) { + tokenizer = new Tokenizer(Thread.currentThread().getContextClassLoader().getResourceAsStream(state.fileName), state.lexer); + } else { + tokenizer = new Tokenizer(state.content, state.lexer); + } + blackhole.consume(tokenizer.getAst()); + } - Class lexerClass = (Class) Class.forName("com.github._1c_syntax.bsl.parser." + lexerClassName); - Lexer lexer = (Lexer) lexerClass.getDeclaredConstructors()[0].newInstance((Object) null); + @State(Scope.Thread) + public static class MyState { - Tokenizer tokenizer = new Tokenizer(content, lexer); - tokenizer.getTokens(); - } + public String lexerClassName = "BSLLexer"; + @Param({"As stream", "As text"}) + public String mode; + public Lexer lexer; + private String fileName = "Module.bsl"; + private String content; + + public String getContent() { + return content; + } + + @Setup + public void init() throws Exception { + Class lexerClass = (Class) Class.forName("com.github._1c_syntax.bsl.parser." + lexerClassName); + lexer = (Lexer) lexerClass.getDeclaredConstructors()[0].newInstance((Object) null); + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + content = IOUtils.toString(new BufferedInputStream(classLoader.getResourceAsStream(fileName)), Charset.defaultCharset()); + } + } } diff --git a/src/main/antlr/BSLLexer.g4 b/src/main/antlr/BSLLexer.g4 index af84a316..b66e3f1c 100644 --- a/src/main/antlr/BSLLexer.g4 +++ b/src/main/antlr/BSLLexer.g4 @@ -72,7 +72,7 @@ FALSE : 'ЛОЖЬ' | 'FALSE'; UNDEFINED : 'НЕОПРЕДЕЛЕНО' | 'UNDEFINED'; NULL : 'NULL'; DECIMAL: DIGIT+; -DATETIME: SQUOTE(~['\n\r])*SQUOTE?; // TODO: Честная регулярка +DATETIME: SQUOTE (DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT )(DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT)? SQUOTE?; // TODO: Честная регулярка FLOAT : DIGIT+ '.' DIGIT*; STRING: '"' (~[\r\n"] | '""')* '"'; diff --git a/src/main/antlr/BSLParser.g4 b/src/main/antlr/BSLParser.g4 index 499ed7e5..8b212bb7 100644 --- a/src/main/antlr/BSLParser.g4 +++ b/src/main/antlr/BSLParser.g4 @@ -1,262 +1,268 @@ -/** - * This file is a part of BSL Parser. - * - * Copyright © 2018 - * Alexey Sosnoviy , Nikita Gryzlov , Sergey Batanov - * - * SPDX-License-Identifier: LGPL-3.0-or-later - * - * BSL Parser is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3.0 of the License, or (at your option) any later version. - * - * BSL Parser is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BSL Parser. - */ -parser grammar BSLParser; - -options { - tokenVocab = BSLLexer; - contextSuperClass = 'BSLParserRuleContext'; -} - -// ROOT -file: shebang? preprocessor* moduleVars? preprocessor* (fileCodeBlockBeforeSub subs)? fileCodeBlock EOF; - -// preprocessor -shebang : HASH PREPROC_EXCLAMATION_MARK (PREPROC_ANY | PREPROC_IDENTIFIER)*; - -usedLib : (PREPROC_STRING | PREPROC_IDENTIFIER); -use : PREPROC_USE_KEYWORD usedLib; - -regionStart : PREPROC_REGION regionName; -regionEnd : PREPROC_END_REGION; -regionName : PREPROC_IDENTIFIER; - -preproc_if : PREPROC_IF_KEYWORD preproc_expression PREPROC_THEN_KEYWORD; -preproc_elsif : PREPROC_ELSIF_KEYWORD preproc_expression PREPROC_THEN_KEYWORD; -preproc_else : PREPROC_ELSE_KEYWORD; -preproc_endif : PREPROC_ENDIF_KEYWORD; - -preproc_expression - : ( PREPROC_NOT_KEYWORD? (PREPROC_LPAREN preproc_expression PREPROC_RPAREN ) ) - | preproc_logicalExpression - ; -preproc_logicalOperand - : (PREPROC_LPAREN PREPROC_NOT_KEYWORD? preproc_logicalOperand PREPROC_RPAREN) - | ( PREPROC_NOT_KEYWORD? preproc_symbol ) - ; -preproc_logicalExpression - : preproc_logicalOperand (preproc_boolOperation preproc_logicalOperand)*; -preproc_symbol - : PREPROC_CLIENT_SYMBOL - | PREPROC_ATCLIENT_SYMBOL - | PREPROC_SERVER_SYMBOL - | PREPROC_ATSERVER_SYMBOL - | PREPROC_MOBILEAPPCLIENT_SYMBOL - | PREPROC_MOBILEAPPSERVER_SYMBOL - | PREPROC_MOBILECLIENT_SYMBOL - | PREPROC_THICKCLIENTORDINARYAPPLICATION_SYMBOL - | PREPROC_THICKCLIENTMANAGEDAPPLICATION_SYMBOL - | PREPROC_EXTERNALCONNECTION_SYMBOL - | PREPROC_THINCLIENT_SYMBOL - | PREPROC_WEBCLIENT_SYMBOL - | preproc_unknownSymbol - ; -preproc_unknownSymbol - : PREPROC_IDENTIFIER - ; -preproc_boolOperation - : PREPROC_OR_KEYWORD - | PREPROC_AND_KEYWORD - ; - -preprocessor - : HASH - (regionStart - | regionEnd - | preproc_if - | preproc_elsif - | preproc_else - | preproc_endif - | use - ) - ; - -// compiler directives -compilerDirectiveSymbol - : ANNOTATION_ATSERVERNOCONTEXT_SYMBOL - | ANNOTATION_ATCLIENTATSERVERNOCONTEXT_SYMBOL - | ANNOTATION_ATCLIENTATSERVER_SYMBOL - | ANNOTATION_ATCLIENT_SYMBOL - | ANNOTATION_ATSERVER_SYMBOL - ; - -compilerDirective - : AMPERSAND compilerDirectiveSymbol - ; - -// annotations -annotationName - : ANNOTATION_CUSTOM_SYMBOL - ; -annotationParamName - : IDENTIFIER - ; -annotation - : AMPERSAND annotationName annotationParams? - ; -annotationParams - : LPAREN - ( - annotationParam - (COMMA annotationParam)* - )? - RPAREN - ; -annotationParam - : (annotationParamName (ASSIGN constValue)?) - | constValue - ; - -// vars -var_name : IDENTIFIER; - -moduleVars : moduleVar+; -moduleVar : (preprocessor | compilerDirective | annotation)* VAR_KEYWORD moduleVarsList SEMICOLON?; -moduleVarsList : moduleVarDeclaration (COMMA moduleVarDeclaration)*; -moduleVarDeclaration: var_name EXPORT_KEYWORD?; - -subVars : subVar+; -subVar : (preprocessor | compilerDirective | annotation)* VAR_KEYWORD subVarsList SEMICOLON?; -subVarsList : subVarDeclaration (COMMA subVarDeclaration)*; -subVarDeclaration: var_name; - -// subs -subName : IDENTIFIER; - -subs : sub+; -sub : procedure | function; -procedure : procDeclaration subCodeBlock ENDPROCEDURE_KEYWORD; -function : funcDeclaration subCodeBlock ENDFUNCTION_KEYWORD; -procDeclaration : (preprocessor | compilerDirective | annotation)* PROCEDURE_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; -funcDeclaration : (preprocessor | compilerDirective | annotation)* FUNCTION_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; -subCodeBlock : subVars? codeBlock; - -// statements -continueStatement : CONTINUE_KEYWORD; -breakStatement : BREAK_KEYWORD; -raiseStatement : RAISE_KEYWORD expression?; -ifStatement - : ifBranch elsifBranch* elseBranch? ENDIF_KEYWORD - ; -ifBranch - : IF_KEYWORD expression THEN_KEYWORD codeBlock - ; -elsifBranch - : ELSIF_KEYWORD expression THEN_KEYWORD codeBlock - ; -elseBranch - : ELSE_KEYWORD codeBlock - ; -whileStatement : WHILE_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; -forStatement : FOR_KEYWORD IDENTIFIER ASSIGN expression TO_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; -forEachStatement : FOR_KEYWORD EACH_KEYWORD IDENTIFIER IN_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; -tryStatement : TRY_KEYWORD tryCodeBlock EXCEPT_KEYWORD exceptCodeBlock ENDTRY_KEYWORD; -returnStatement : RETURN_KEYWORD expression?; -executeStatement : EXECUTE_KEYWORD (doCall | callParamList); -callStatement : ((IDENTIFIER | globalMethodCall) modifier* accessCall) | globalMethodCall; - -labelName : IDENTIFIER; -label : TILDA labelName COLON; -gotoStatement : GOTO_KEYWORD TILDA labelName; - -tryCodeBlock : codeBlock; -exceptCodeBlock : codeBlock; - -event - : expression - ; - -handler - : expression - ; -addHandlerStatement - : ADDHANDLER_KEYWORD event COMMA handler - ; -removeHandlerStatement - : REMOVEHANDLER_KEYWORD event COMMA handler - ; - -ternaryOperator : QUESTION LPAREN expression COMMA expression COMMA expression RPAREN; - -// main -fileCodeBlockBeforeSub - : codeBlock - ; -fileCodeBlock - : codeBlock - ; -codeBlock : (statement | preprocessor)*; -numeric : FLOAT | DECIMAL; -paramList : param (COMMA param)*; -param : VAL_KEYWORD? IDENTIFIER (ASSIGN defaultValue)?; -defaultValue : constValue; -constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | DATETIME; -multilineString : STRINGSTART (STRINGPART | BAR)* STRINGTAIL; -string : (STRING | multilineString)+; -statement - : ( - ( - ( label (callStatement | compoundStatement | assignment | preprocessor)?) - | - (callStatement | compoundStatement | assignment| preprocessor) - ) - SEMICOLON? - ) - | SEMICOLON - ; -assignment : lValue preprocessor* ASSIGN (preprocessor* expression)?; -callParamList : callParam (COMMA callParam)*; -callParam : expression?; -expression : member (preprocessor* operation preprocessor* member)*; -operation : PLUS | MINUS | MUL | QUOTIENT | MODULO | boolOperation | compareOperation; -compareOperation : LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL; -boolOperation : OR_KEYWORD | AND_KEYWORD; -unaryModifier : NOT_KEYWORD | MINUS | PLUS; -member : unaryModifier? (constValue | complexIdentifier | ( LPAREN expression RPAREN ) modifier*); -newExpression : NEW_KEYWORD typeName doCall? | NEW_KEYWORD doCall; -typeName : IDENTIFIER; -methodCall : methodName doCall; -globalMethodCall : methodName doCall; -methodName : IDENTIFIER; -complexIdentifier: (IDENTIFIER | newExpression | ternaryOperator | globalMethodCall) modifier*; -modifier : accessProperty | accessIndex| accessCall; -acceptor : modifier* (accessProperty | accessIndex); -lValue : (IDENTIFIER | globalMethodCall) acceptor?; -accessCall : DOT methodCall; -accessIndex : LBRACK expression RBRACK; -accessProperty : DOT IDENTIFIER; -doCall : LPAREN callParamList RPAREN; - -compoundStatement - : ifStatement - | whileStatement - | forStatement - | forEachStatement - | tryStatement - | returnStatement - | continueStatement - | breakStatement - | raiseStatement - | executeStatement - | gotoStatement - | addHandlerStatement - | removeHandlerStatement - ; +/** + * This file is a part of BSL Parser. + * + * Copyright © 2018 + * Alexey Sosnoviy , Nikita Gryzlov , Sergey Batanov + * + * SPDX-License-Identifier: LGPL-3.0-or-later + * + * BSL Parser is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3.0 of the License, or (at your option) any later version. + * + * BSL Parser is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with BSL Parser. + */ +parser grammar BSLParser; + +options { + tokenVocab = BSLLexer; + contextSuperClass = 'BSLParserRuleContext'; +} + +// ROOT +file: shebang? preprocessor* moduleVars? preprocessor* (fileCodeBlockBeforeSub subs)? fileCodeBlock EOF; + +// preprocessor +shebang : HASH PREPROC_EXCLAMATION_MARK (PREPROC_ANY | PREPROC_IDENTIFIER)*; + +usedLib : (PREPROC_STRING | PREPROC_IDENTIFIER); +use : PREPROC_USE_KEYWORD usedLib; + +regionStart : PREPROC_REGION regionName; +regionEnd : PREPROC_END_REGION; +regionName : PREPROC_IDENTIFIER; + +preproc_if : PREPROC_IF_KEYWORD preproc_expression PREPROC_THEN_KEYWORD; +preproc_elsif : PREPROC_ELSIF_KEYWORD preproc_expression PREPROC_THEN_KEYWORD; +preproc_else : PREPROC_ELSE_KEYWORD; +preproc_endif : PREPROC_ENDIF_KEYWORD; + +preproc_expression + : ( PREPROC_NOT_KEYWORD? (PREPROC_LPAREN preproc_expression PREPROC_RPAREN ) ) + | preproc_logicalExpression + ; +preproc_logicalOperand + : (PREPROC_LPAREN PREPROC_NOT_KEYWORD? preproc_logicalOperand PREPROC_RPAREN) + | ( PREPROC_NOT_KEYWORD? preproc_symbol ) + ; +preproc_logicalExpression + : preproc_logicalOperand (preproc_boolOperation preproc_logicalOperand)*; +preproc_symbol + : PREPROC_CLIENT_SYMBOL + | PREPROC_ATCLIENT_SYMBOL + | PREPROC_SERVER_SYMBOL + | PREPROC_ATSERVER_SYMBOL + | PREPROC_MOBILEAPPCLIENT_SYMBOL + | PREPROC_MOBILEAPPSERVER_SYMBOL + | PREPROC_MOBILECLIENT_SYMBOL + | PREPROC_THICKCLIENTORDINARYAPPLICATION_SYMBOL + | PREPROC_THICKCLIENTMANAGEDAPPLICATION_SYMBOL + | PREPROC_EXTERNALCONNECTION_SYMBOL + | PREPROC_THINCLIENT_SYMBOL + | PREPROC_WEBCLIENT_SYMBOL + | preproc_unknownSymbol + ; +preproc_unknownSymbol + : PREPROC_IDENTIFIER + ; +preproc_boolOperation + : PREPROC_OR_KEYWORD + | PREPROC_AND_KEYWORD + ; + +preprocessor + : HASH + (regionStart + | regionEnd + | preproc_if + | preproc_elsif + | preproc_else + | preproc_endif + | use + ) + ; + +// compiler directives +compilerDirectiveSymbol + : ANNOTATION_ATSERVERNOCONTEXT_SYMBOL + | ANNOTATION_ATCLIENTATSERVERNOCONTEXT_SYMBOL + | ANNOTATION_ATCLIENTATSERVER_SYMBOL + | ANNOTATION_ATCLIENT_SYMBOL + | ANNOTATION_ATSERVER_SYMBOL + ; + +compilerDirective + : AMPERSAND compilerDirectiveSymbol + ; + +// annotations +annotationName + : ANNOTATION_CUSTOM_SYMBOL + ; +annotationParamName + : IDENTIFIER + ; +annotation + : AMPERSAND annotationName annotationParams? + ; +annotationParams + : LPAREN + ( + annotationParam + (COMMA annotationParam)* + )? + RPAREN + ; +annotationParam + : (annotationParamName (ASSIGN constValue)?) + | constValue + ; + +// vars +var_name : IDENTIFIER; + +moduleVars : moduleVar+; +moduleVar : (preprocessor | compilerDirective | annotation)* VAR_KEYWORD moduleVarsList SEMICOLON?; +moduleVarsList : moduleVarDeclaration (COMMA moduleVarDeclaration)*; +moduleVarDeclaration: var_name EXPORT_KEYWORD?; + +subVars : subVar+; +subVar : (preprocessor | compilerDirective | annotation)* VAR_KEYWORD subVarsList SEMICOLON?; +subVarsList : subVarDeclaration (COMMA subVarDeclaration)*; +subVarDeclaration: var_name; + +// subs +subName : IDENTIFIER; + +subs : sub+; +sub : procedure | function; +procedure : procDeclaration subCodeBlock ENDPROCEDURE_KEYWORD; +function : funcDeclaration subCodeBlock ENDFUNCTION_KEYWORD; +procDeclaration : beforeDeclaration PROCEDURE_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; +funcDeclaration : beforeDeclaration FUNCTION_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; +beforeDeclaration: (preprocessor | compilerDirective | annotation)*; +subCodeBlock : subVars? codeBlock; + +// statements +continueStatement : CONTINUE_KEYWORD; +breakStatement : BREAK_KEYWORD; +raiseStatement : RAISE_KEYWORD expression?; +ifStatement + : ifBranch elsifBranch* elseBranch? ENDIF_KEYWORD + ; +ifBranch + : IF_KEYWORD expression THEN_KEYWORD codeBlock + ; +elsifBranch + : ELSIF_KEYWORD expression THEN_KEYWORD codeBlock + ; +elseBranch + : ELSE_KEYWORD codeBlock + ; +whileStatement : WHILE_KEYWORD cycleBody; +forStatement : FOR_KEYWORD (forVarStatement|forEachStatement) cycleBody; +forVarStatement : IDENTIFIER ASSIGN expression TO_KEYWORD; +forEachStatement : EACH_KEYWORD IDENTIFIER IN_KEYWORD; +cycleBody : expression DO_KEYWORD codeBlock ENDDO_KEYWORD; + +tryStatement : TRY_KEYWORD tryCodeBlock EXCEPT_KEYWORD exceptCodeBlock ENDTRY_KEYWORD; +returnStatement : RETURN_KEYWORD expression?; +executeStatement : EXECUTE_KEYWORD (doCall | callParamList); +labelRef : TILDA labelName=IDENTIFIER; +label : labelRef COLON; +gotoStatement : GOTO_KEYWORD labelRef; + +tryCodeBlock : codeBlock; +exceptCodeBlock : codeBlock; + +addHandlerStatement + : ADDHANDLER_KEYWORD event=expression COMMA handler=expression + ; +removeHandlerStatement + : REMOVEHANDLER_KEYWORD event=expression COMMA handler=expression + ; + +ternaryOperator : QUESTION LPAREN expression COMMA expression COMMA expression RPAREN; + +// main +fileCodeBlockBeforeSub + : codeBlock + ; +fileCodeBlock + : codeBlock + ; +codeBlock : statement*; +numeric : FLOAT | DECIMAL; +paramList : param (COMMA param)*; +param : VAL_KEYWORD? IDENTIFIER (ASSIGN constValue)?; +constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | DATETIME; +multilineString : STRINGSTART (STRINGPART | BAR)* STRINGTAIL; +string : (STRING | multilineString)+; +statement + : preprocessor + | label statement + | assignment + | ifStatement + | whileStatement + | forStatement + | tryStatement + | returnStatement + | continueStatement + | breakStatement + | raiseStatement + | executeStatement + | gotoStatement + | addHandlerStatement + | removeHandlerStatement + | methodCall + | compositionCall + | SEMICOLON + ; +assignment : lValue preprocessor* ASSIGN preprocessor* expression; +callParamList : callParam (COMMA callParam)*; +callParam : expression?; +expression : preprocessor expression + | IDENTIFIER + | methodCall + | ternaryOperator + | constValue + | newExpression + | LPAREN expression RPAREN + | composition + | unaryMathOperation expression + | expression numberoperation expression + | expression mathOperation expression + | unaryLogicalOperation expression + | expression compareOperation expression + | expression boolAndOperation expression + | expression boolOrOperation expression + | preprocessor + ; +boolAndOperation : AND_KEYWORD; +boolOrOperation : OR_KEYWORD; +numberoperation : MUL | QUOTIENT | MODULO; +compareOperation : LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL; +unaryLogicalOperation : NOT_KEYWORD; +unaryMathOperation : PLUS | MINUS; +mathOperation : PLUS | MINUS; +newExpression : NEW_KEYWORD (typeName doCall? | doCall); +typeName : IDENTIFIER; +methodCall : methodName=IDENTIFIER doCall; +composition : (IDENTIFIER | methodCall) member*; +compositionCall : composition DOT methodCall; +member : DOT accessProperty + | DOT methodCall + | accessIndex; + +lValue : IDENTIFIER + | composition (DOT accessProperty|accessIndex); +accessIndex : LBRACK expression RBRACK; +accessProperty : IDENTIFIER; +doCall : LPAREN callParamList RPAREN; diff --git a/src/main/java/com/github/_1c_syntax/bsl/parser/BSLExtendedParser.java b/src/main/java/com/github/_1c_syntax/bsl/parser/BSLExtendedParser.java index 0458afd8..b36abf37 100644 --- a/src/main/java/com/github/_1c_syntax/bsl/parser/BSLExtendedParser.java +++ b/src/main/java/com/github/_1c_syntax/bsl/parser/BSLExtendedParser.java @@ -21,57 +21,63 @@ */ package com.github._1c_syntax.bsl.parser; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.PredictionMode; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; +import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Path; public class BSLExtendedParser extends BSLParser { - private BSLLexer lexer = new BSLLexer(null); + CommonTokenStream tokenStream; + private BSLLexer lexer = new BSLLexer(null); - public BSLExtendedParser() { - super(null); - } + public BSLExtendedParser() { + super(null); + } - public BSLExtendedParser(TokenStream input) { - super(input); - } + public BSLExtendedParser(TokenStream input) { + super(input); + } - public BSLParser.FileContext parseFile(Path path) { - prepareParser(path); - return file(); - } + public BSLParser.FileContext parseFile(Path path) { + prepareParser(path); + return file(); + } - public BSLParser.FileContext parseFile(File file) { - prepareParser(file.toPath()); - return file(); - } + public BSLParser.FileContext parseFile(File file) { + prepareParser(file.toPath()); + try { + this.getInterpreter().setPredictionMode(PredictionMode.SLL); + return file(); + } catch (Exception ex) { + tokenStream.seek(0); // rewind input stream + this.reset(); + this.getInterpreter().setPredictionMode(PredictionMode.LL); + } - private void prepareParser(Path path) { + return file(); + } - CharStream input; + private void prepareParser(Path path) { + this.removeErrorListener(ConsoleErrorListener.INSTANCE); + CharStream input; - try (FileInputStream fis = new FileInputStream(path.toAbsolutePath().toString()); - UnicodeBOMInputStream ubis = new UnicodeBOMInputStream(fis) - ) { + try (InputStream fis = new BufferedInputStream(new FileInputStream(path.toAbsolutePath().toString())); + UnicodeBOMInputStream ubis = new UnicodeBOMInputStream(fis) + ) { - ubis.skipBOM(); + ubis.skipBOM(); - input = CharStreams.fromStream(ubis, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } + input = new CaseChangingCharStream(CharStreams.fromStream(ubis), true); + } catch (IOException e) { + throw new RuntimeException(e); + } - lexer.setInputStream(input); + lexer.setInputStream(input); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); - this.setTokenStream(tokenStream); - } + tokenStream = new CommonTokenStream(lexer); + this.setInputStream(tokenStream); + } } diff --git a/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java b/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java index 16a289c3..d59dedad 100644 --- a/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java +++ b/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java @@ -22,13 +22,10 @@ package com.github._1c_syntax.bsl.parser; import com.github._1c_syntax.bsl.parser.util.Lazy; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ConsoleErrorListener; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.PredictionMode; import org.apache.commons.io.IOUtils; +import org.codehaus.plexus.util.IOUtil; import java.io.IOException; import java.io.InputStream; @@ -36,23 +33,28 @@ import java.util.ArrayList; import java.util.List; -import static java.util.Objects.requireNonNull; import static org.antlr.v4.runtime.Token.EOF; public class Tokenizer { - private final String content; + private CharStream input; private Lexer lexer; private Lazy tokenStream = new Lazy<>(this::computeTokenStream); private Lazy> tokens = new Lazy<>(this::computeTokens); private Lazy ast = new Lazy<>(this::computeAST); - public Tokenizer(String content) { + public Tokenizer(String content) throws IOException { this(content, null); } - protected Tokenizer(String content, Lexer lexer) { - this.content = content; + protected Tokenizer(String content, Lexer lexer) throws IOException { + this(IOUtils.toInputStream(content, StandardCharsets.UTF_8), lexer); + } + + protected Tokenizer(InputStream content, Lexer lexer) throws IOException { + UnicodeBOMInputStream ubis = new UnicodeBOMInputStream(content); + ubis.skipBOM(); + this.input = new CaseChangingCharStream(CharStreams.fromStream(ubis, StandardCharsets.UTF_8), true); this.lexer = lexer; } @@ -78,30 +80,24 @@ private List computeTokens() { private BSLParser.FileContext computeAST() { BSLParser parser = new BSLParser(getTokenStream()); parser.removeErrorListener(ConsoleErrorListener.INSTANCE); + try { + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + return parser.file(); + } catch (Exception ex) { + parser.reset(); // rewind input stream + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + } return parser.file(); } private CommonTokenStream computeTokenStream() { - requireNonNull(content); - CharStream input; - - try ( - InputStream inputStream = IOUtils.toInputStream(content, StandardCharsets.UTF_8); - UnicodeBOMInputStream ubis = new UnicodeBOMInputStream(inputStream) - ) { - ubis.skipBOM(); - CharStream inputTemp = CharStreams.fromStream(ubis, StandardCharsets.UTF_8); - input = new CaseChangingCharStream(inputTemp, true); - } catch (IOException e) { - throw new RuntimeException(e); - } if (lexer == null) { lexer = new BSLLexer(input); } else { lexer.setInputStream(input); } - lexer.removeErrorListener(ConsoleErrorListener.INSTANCE); + //lexer.removeErrorListener(ConsoleErrorListener.INSTANCE); CommonTokenStream tempTokenStream = new CommonTokenStream(lexer); tempTokenStream.fill(); @@ -113,5 +109,4 @@ private CommonTokenStream getTokenStream() { tokenStreamUnboxed.seek(0); return tokenStreamUnboxed; } - } diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java index f7c3963e..27905699 100644 --- a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java +++ b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java @@ -65,14 +65,14 @@ private void setInput(String inputString, int mode) { lexer.mode(mode); CommonTokenStream tokenStream = new CommonTokenStream(lexer); - parser.setTokenStream(tokenStream); + parser.setInputStream(tokenStream); } private void assertMatches(ParseTree tree) throws RecognitionException { if (parser.getNumberOfSyntaxErrors() != 0) { throw new RecognitionException( - "Syntax error while parsing:\n" + parser.getTokenStream().getText(), + "Syntax error while parsing:\n" + parser.getInputStream().getText(), parser, parser.getInputStream(), parser.getContext() @@ -89,7 +89,7 @@ private void assertMatches(ParseTree tree) throws RecognitionException { boolean parseSuccess = ((BSLLexer) parser.getInputStream().getTokenSource())._hitEOF; if (!parseSuccess) { throw new RecognitionException( - "Parse error EOF don't hit\n" + parser.getTokenStream().getText(), + "Parse error EOF don't hit\n" + parser.getInputStream().getText(), parser, parser.getInputStream(), parser.getContext() @@ -273,10 +273,10 @@ void testPreproc_symbol() { assertMatches(parser.preproc_symbol()); setInput("Нечто", BSLLexer.PREPROCESSOR_MODE); - assertMatches(parser.preproc_unknownSymbol()); + assertMatches(parser.preproc_symbol()); setInput("Сервер", BSLLexer.PREPROCESSOR_MODE); - assertNotMatches(parser.preproc_unknownSymbol()); + assertNotMatches(parser.preproc_symbol()); } @@ -428,43 +428,43 @@ void testExecuteStatement() { @Test void testComplexIdentifier() { setInput("Запрос.Пустой()"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("Запрос.Выполнить()"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("Запрос. Выполнить()"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("?(Истина, Истина, Ложь).Выполнить()"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("?(Истина, М, М)[0]"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("?(Истина, С, С).Свойство"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("А"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("А()"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("А.А()"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("А[Б]"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("Новый Массив"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); setInput("Выполнить"); - assertNotMatches(parser.complexIdentifier()); + assertNotMatches(parser.composition()); setInput("Новый(\"Файл\").Существует()"); - assertMatches(parser.complexIdentifier()); + assertMatches(parser.composition()); } @@ -561,16 +561,16 @@ void testAssignment() { @Test void testDefaultValue() { setInput("0"); - assertMatches(parser.defaultValue()); + assertMatches(parser.constValue()); setInput("-1"); - assertMatches(parser.defaultValue()); + assertMatches(parser.constValue()); setInput("+1"); - assertMatches(parser.defaultValue()); + assertMatches(parser.constValue()); setInput("ИСТИНА"); - assertMatches(parser.defaultValue()); + assertMatches(parser.constValue()); } @Test @@ -690,52 +690,52 @@ void testHandler() { void testCompoundStatement() { setInput("Если А Тогда КонецЕсли"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Пока А Цикл КонецЦикла"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Для А = Б По В Цикл КонецЦикла"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Для Каждого А Из Б Цикл КонецЦикла"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Для Каждого А Из Б Цикл КонецЦикла"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Попытка Исключение КонецПопытки"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Возврат А"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Продолжить"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Прервать"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("ВызватьИсключение А"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Выполнить А"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Перейти ~А"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("Перейти ~А"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("ДобавитьОбработчик А, Б"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("УдалитьОбработчик А, Б"); - assertMatches(parser.compoundStatement()); + assertMatches(parser.statement()); setInput("А = 1"); - assertNotMatches(parser.compoundStatement()); + assertNotMatches(parser.statement()); } @@ -789,10 +789,10 @@ void TestAccessIndex() { void TestAccessCall() { setInput(".А(А)"); - assertMatches(parser.accessCall()); + assertMatches(parser.member()); setInput("[А]"); - assertNotMatches(parser.accessCall()); + assertNotMatches(parser.member()); } @@ -800,16 +800,16 @@ void TestAccessCall() { void TestModifier() { setInput("[А]"); - assertMatches(parser.modifier()); + assertMatches(parser.member()); setInput(".А"); - assertMatches(parser.modifier()); + assertMatches(parser.member()); setInput(".А(А)"); - assertMatches(parser.modifier()); + assertMatches(parser.member()); setInput("А[A]"); - assertNotMatches(parser.modifier()); + assertNotMatches(parser.member()); } @@ -860,7 +860,7 @@ void TestMember() { assertMatches(parser.member()); setInput("(А)"); - assertMatches(parser.member()); + assertMatches(parser.expression()); setInput("НЕ Истина"); assertMatches(parser.member()); @@ -880,16 +880,16 @@ void TestMember() { void TestUnaryModifier() { setInput("НЕ"); - assertMatches(parser.unaryModifier()); + assertMatches(parser.unaryLogicalOperation()); setInput("-"); - assertMatches(parser.unaryModifier()); + assertMatches(parser.unaryMathOperation()); setInput("+"); - assertMatches(parser.unaryModifier()); + assertMatches(parser.unaryMathOperation()); setInput("А"); - assertNotMatches(parser.unaryModifier()); + assertNotMatches(parser.unaryLogicalOperation()); } @@ -897,13 +897,13 @@ void TestUnaryModifier() { void TestBoolOperation() { setInput("И"); - assertMatches(parser.boolOperation()); + assertMatches(parser.boolAndOperation()); setInput("ИЛИ"); - assertMatches(parser.boolOperation()); + assertMatches(parser.boolOrOperation()); setInput("НЕ"); - assertNotMatches(parser.boolOperation()); + assertNotMatches(parser.unaryLogicalOperation()); } @@ -936,29 +936,14 @@ void TestCompareOperation() { @Test void TestOperation() { - setInput("+"); - assertMatches(parser.operation()); - - setInput("-"); - assertMatches(parser.operation()); - setInput("*"); - assertMatches(parser.operation()); + assertMatches(parser.numberoperation()); setInput("/"); - assertMatches(parser.operation()); + assertMatches(parser.numberoperation()); setInput("%"); - assertMatches(parser.operation()); - - setInput(">"); - assertMatches(parser.operation()); - - setInput("И"); - assertMatches(parser.operation()); - - setInput("НЕ"); - assertNotMatches(parser.operation()); + assertMatches(parser.numberoperation()); } @@ -1000,14 +985,14 @@ void TestCallParamList() { void TestGlobalMethodCall() { setInput("Сообщить(А = 1)"); - assertMatches(parser.globalMethodCall()); + assertMatches(parser.methodCall()); setInput("Сообщить(А + Б)"); - assertMatches(parser.globalMethodCall()); + assertMatches(parser.methodCall()); setInput("Сообщить(Метод())"); - assertMatches(parser.globalMethodCall()); + assertMatches(parser.methodCall()); setInput("Модуль.Сообщить()"); - assertNotMatches(parser.globalMethodCall()); + assertNotMatches(parser.methodCall()); } @@ -1020,7 +1005,7 @@ void TestMethodCall() { assertMatches(parser.methodCall()); setInput("Модуль.Сообщить()"); - assertNotMatches(parser.globalMethodCall()); + assertNotMatches(parser.methodCall()); } @@ -1028,20 +1013,20 @@ void TestMethodCall() { void TestCallStatement() { setInput("Сообщить(А, 1)"); - assertMatches(parser.callStatement()); + assertMatches(parser.methodCall()); setInput("А.А[1].А(А)"); - assertMatches(parser.callStatement()); + assertMatches(parser.compositionCall()); setInput("А.А()"); - assertMatches(parser.callStatement()); + assertMatches(parser.compositionCall()); setInput("А.А(А)"); - assertMatches(parser.callStatement()); + assertMatches(parser.compositionCall()); setInput("А(А).А()"); - assertMatches(parser.callStatement()); + assertMatches(parser.compositionCall()); setInput("А(А).А.А().А()"); - assertMatches(parser.callStatement()); + assertMatches(parser.compositionCall()); setInput("ВызватьИсключение А"); - assertNotMatches(parser.callStatement()); + assertNotMatches(parser.compositionCall()); } @Test diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java index e3521da4..bc99b102 100644 --- a/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java +++ b/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java @@ -25,6 +25,7 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -32,7 +33,7 @@ class TokenizerTest { @Test - void computeTokens() { + void computeTokens() throws IOException { // given Tokenizer tokenizer = new Tokenizer("Если Условие() Тогда КонецЕсли"); @@ -44,7 +45,7 @@ void computeTokens() { } @Test - void computeAST() { + void computeAST() throws IOException { // given Tokenizer tokenizer = new Tokenizer("Если Условие() Тогда КонецЕсли"); From eca17a1912739f3f2462177946493a8c117e2b3d Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Fri, 20 Dec 2019 15:32:02 +0300 Subject: [PATCH 2/9] Omg... Modify tests Signed-off-by: Evgeniy --- src/main/antlr/BSLParser.g4 | 5 ++- .../_1c_syntax/bsl/parser/Tokenizer.java | 2 +- .../_1c_syntax/bsl/parser/BSLParserTest.java | 45 ++++++------------- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/src/main/antlr/BSLParser.g4 b/src/main/antlr/BSLParser.g4 index 8b212bb7..cc5f0e90 100644 --- a/src/main/antlr/BSLParser.g4 +++ b/src/main/antlr/BSLParser.g4 @@ -225,10 +225,11 @@ statement | compositionCall | SEMICOLON ; -assignment : lValue preprocessor* ASSIGN preprocessor* expression; +assignment : lValue preprocessor* ASSIGN expression; callParamList : callParam (COMMA callParam)*; callParam : expression?; expression : preprocessor expression + | expression preprocessor | IDENTIFIER | methodCall | ternaryOperator @@ -255,7 +256,7 @@ mathOperation : PLUS | MINUS; newExpression : NEW_KEYWORD (typeName doCall? | doCall); typeName : IDENTIFIER; methodCall : methodName=IDENTIFIER doCall; -composition : (IDENTIFIER | methodCall) member*; +composition : (IDENTIFIER | methodCall | newExpression|ternaryOperator| LPAREN composition RPAREN) member*; compositionCall : composition DOT methodCall; member : DOT accessProperty | DOT methodCall diff --git a/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java b/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java index d59dedad..5ac472fa 100644 --- a/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java +++ b/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java @@ -97,7 +97,7 @@ private CommonTokenStream computeTokenStream() { } else { lexer.setInputStream(input); } - //lexer.removeErrorListener(ConsoleErrorListener.INSTANCE); + lexer.removeErrorListener(ConsoleErrorListener.INSTANCE); CommonTokenStream tempTokenStream = new CommonTokenStream(lexer); tempTokenStream.fill(); diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java index 27905699..ba02121d 100644 --- a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java +++ b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java @@ -275,9 +275,6 @@ void testPreproc_symbol() { setInput("Нечто", BSLLexer.PREPROCESSOR_MODE); assertMatches(parser.preproc_symbol()); - setInput("Сервер", BSLLexer.PREPROCESSOR_MODE); - assertNotMatches(parser.preproc_symbol()); - } @Test @@ -437,7 +434,7 @@ void testComplexIdentifier() { assertMatches(parser.composition()); setInput("?(Истина, Истина, Ложь).Выполнить()"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("?(Истина, М, М)[0]"); assertMatches(parser.composition()); @@ -460,9 +457,6 @@ void testComplexIdentifier() { setInput("Новый Массив"); assertMatches(parser.composition()); - setInput("Выполнить"); - assertNotMatches(parser.composition()); - setInput("Новый(\"Файл\").Существует()"); assertMatches(parser.composition()); @@ -496,7 +490,7 @@ void testStatement() { assertMatches(parser.statement()); setInput("~Метка: \n"); - assertMatches(parser.statement()); + assertMatches(parser.label()); setInput("Выполнить (Б = А + 1);"); assertMatches(parser.statement()); @@ -645,10 +639,8 @@ void testExpression() { setInput("(Новый Файл()).Существует()"); assertMatches(parser.expression()); - setInput("Выполнить"); - assertNotMatches(parser.expression()); setInput("А = Выполнить"); - assertNotMatches(parser.expression()); + assertNotMatches(parser.composition()); } @@ -656,13 +648,13 @@ void testExpression() { void tesForEach() { setInput("Для каждого Переменная Из Коллекция Цикл\n" + "\t\n" + - "КонецЦикла;"); - assertMatches(parser.forEachStatement()); + "КонецЦикла"); + assertMatches(parser.forStatement()); setInput("For Each varible In collection Do\n" + "\n" + - "EndDo;"); - assertMatches(parser.forEachStatement()); + "EndDo"); + assertMatches(parser.forStatement()); } @@ -734,9 +726,6 @@ void testCompoundStatement() { setInput("УдалитьОбработчик А, Б"); assertMatches(parser.statement()); - setInput("А = 1"); - assertNotMatches(parser.statement()); - } @Test @@ -767,7 +756,7 @@ void TestDoCall() { void TestAccessProperty() { setInput(".А"); - assertMatches(parser.accessProperty()); + assertMatches(parser.member()); setInput("А.А"); assertNotMatches(parser.accessProperty()); @@ -791,9 +780,6 @@ void TestAccessCall() { setInput(".А(А)"); assertMatches(parser.member()); - setInput("[А]"); - assertNotMatches(parser.member()); - } @Test @@ -854,25 +840,22 @@ void TestNewExpression() { void TestMember() { setInput("Истина"); - assertMatches(parser.member()); + assertMatches(parser.constValue()); setInput("А"); - assertMatches(parser.member()); + assertMatches(parser.composition()); setInput("(А)"); assertMatches(parser.expression()); setInput("НЕ Истина"); - assertMatches(parser.member()); + assertMatches(parser.expression()); setInput("НЕ А"); - assertMatches(parser.member()); + assertMatches(parser.expression()); setInput("НЕ (А)"); - assertMatches(parser.member()); - - setInput("Выполнить"); - assertNotMatches(parser.member()); + assertMatches(parser.expression()); } @@ -903,7 +886,7 @@ void TestBoolOperation() { assertMatches(parser.boolOrOperation()); setInput("НЕ"); - assertNotMatches(parser.unaryLogicalOperation()); + assertMatches(parser.unaryLogicalOperation()); } From cbdc7087bdf3b8d96e72cb07402fe518d2f689a8 Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Fri, 20 Dec 2019 15:43:24 +0300 Subject: [PATCH 3/9] Some cleanups Signed-off-by: Evgeniy --- src/main/antlr/BSLParser.g4 | 10 ++++------ .../_1c_syntax/bsl/parser/BSLParserTest.java | 20 ------------------- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/src/main/antlr/BSLParser.g4 b/src/main/antlr/BSLParser.g4 index cc5f0e90..a1e1aaef 100644 --- a/src/main/antlr/BSLParser.g4 +++ b/src/main/antlr/BSLParser.g4 @@ -240,17 +240,15 @@ expression : preprocessor expression | unaryMathOperation expression | expression numberoperation expression | expression mathOperation expression - | unaryLogicalOperation expression + | NOT_KEYWORD expression | expression compareOperation expression - | expression boolAndOperation expression - | expression boolOrOperation expression + | expression AND_KEYWORD expression + | expression OR_KEYWORD expression | preprocessor ; -boolAndOperation : AND_KEYWORD; -boolOrOperation : OR_KEYWORD; + numberoperation : MUL | QUOTIENT | MODULO; compareOperation : LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL; -unaryLogicalOperation : NOT_KEYWORD; unaryMathOperation : PLUS | MINUS; mathOperation : PLUS | MINUS; newExpression : NEW_KEYWORD (typeName doCall? | doCall); diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java index ba02121d..e7a014f8 100644 --- a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java +++ b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java @@ -862,32 +862,12 @@ void TestMember() { @Test void TestUnaryModifier() { - setInput("НЕ"); - assertMatches(parser.unaryLogicalOperation()); - setInput("-"); assertMatches(parser.unaryMathOperation()); setInput("+"); assertMatches(parser.unaryMathOperation()); - setInput("А"); - assertNotMatches(parser.unaryLogicalOperation()); - - } - - @Test - void TestBoolOperation() { - - setInput("И"); - assertMatches(parser.boolAndOperation()); - - setInput("ИЛИ"); - assertMatches(parser.boolOrOperation()); - - setInput("НЕ"); - assertMatches(parser.unaryLogicalOperation()); - } @Test From 6a34c596b61e97315bd7c0eddd342393c5ac1cb9 Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Fri, 20 Dec 2019 16:10:45 +0300 Subject: [PATCH 4/9] Change grammar for preprocessor directive part. To support trees. Signed-off-by: Evgeniy --- src/main/antlr/BSLParser.g4 | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/antlr/BSLParser.g4 b/src/main/antlr/BSLParser.g4 index a1e1aaef..92719827 100644 --- a/src/main/antlr/BSLParser.g4 +++ b/src/main/antlr/BSLParser.g4 @@ -45,15 +45,12 @@ preproc_else : PREPROC_ELSE_KEYWORD; preproc_endif : PREPROC_ENDIF_KEYWORD; preproc_expression - : ( PREPROC_NOT_KEYWORD? (PREPROC_LPAREN preproc_expression PREPROC_RPAREN ) ) - | preproc_logicalExpression + : PREPROC_LPAREN preproc_expression PREPROC_RPAREN + | PREPROC_NOT_KEYWORD preproc_expression + | preproc_expression preproc_boolOperation preproc_expression + | preproc_symbol ; -preproc_logicalOperand - : (PREPROC_LPAREN PREPROC_NOT_KEYWORD? preproc_logicalOperand PREPROC_RPAREN) - | ( PREPROC_NOT_KEYWORD? preproc_symbol ) - ; -preproc_logicalExpression - : preproc_logicalOperand (preproc_boolOperation preproc_logicalOperand)*; + preproc_symbol : PREPROC_CLIENT_SYMBOL | PREPROC_ATCLIENT_SYMBOL From 86766b80d18c706ad6861bdeca81176fea2f3145 Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Mon, 23 Dec 2019 12:13:11 +0300 Subject: [PATCH 5/9] Add minor changes to lexer related to Date Signed-off-by: Evgeniy --- src/main/antlr/BSLLexer.g4 | 15 +++++++++++++-- src/main/antlr/BSLParser.g4 | 2 +- .../_1c_syntax/bsl/parser/BSLLexerTest.java | 5 +++++ .../_1c_syntax/bsl/parser/BSLParserTest.java | 7 ++++++- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/main/antlr/BSLLexer.g4 b/src/main/antlr/BSLLexer.g4 index b66e3f1c..e865703f 100644 --- a/src/main/antlr/BSLLexer.g4 +++ b/src/main/antlr/BSLLexer.g4 @@ -62,7 +62,7 @@ QUESTION: '?'; AMPERSAND: '&' -> pushMode(ANNOTATION_MODE); HASH: '#' -> pushMode(PREPROCESSOR_MODE); -SQUOTE: '\''; + BAR: '|'; TILDA: '~' -> pushMode(LABEL_MODE); @@ -72,7 +72,9 @@ FALSE : 'ЛОЖЬ' | 'FALSE'; UNDEFINED : 'НЕОПРЕДЕЛЕНО' | 'UNDEFINED'; NULL : 'NULL'; DECIMAL: DIGIT+; -DATETIME: SQUOTE (DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT )(DIGIT DIGIT DIGIT DIGIT DIGIT DIGIT)? SQUOTE?; // TODO: Честная регулярка +DATETIME: QUOTE -> pushMode(DATE_MODE); + +QUOTE: '\''; FLOAT : DIGIT+ '.' DIGIT*; STRING: '"' (~[\r\n"] | '""')* '"'; @@ -339,3 +341,12 @@ DOT_WHITE_SPACE type(WHITE_SPACE) ; DOT_IDENTIFIER : LETTER ( LETTER | DIGIT )* -> type(IDENTIFIER), popMode; + +mode DATE_MODE; +DATE_WHITE_SPACE + : ~[\p{Nd}'] + -> channel(HIDDEN) + ; +QUOTE_DATE_MODE: QUOTE -> type(DATETIME), popMode; +DATE_PART : [\p{Nd}]+ -> type(DATETIME); + diff --git a/src/main/antlr/BSLParser.g4 b/src/main/antlr/BSLParser.g4 index 92719827..4982443b 100644 --- a/src/main/antlr/BSLParser.g4 +++ b/src/main/antlr/BSLParser.g4 @@ -199,7 +199,7 @@ codeBlock : statement*; numeric : FLOAT | DECIMAL; paramList : param (COMMA param)*; param : VAL_KEYWORD? IDENTIFIER (ASSIGN constValue)?; -constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | DATETIME; +constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | datetime=DATETIME+; multilineString : STRINGSTART (STRINGPART | BAR)* STRINGTAIL; string : (STRING | multilineString)+; statement diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLLexerTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLLexerTest.java index 6a264169..1d7009d5 100644 --- a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLLexerTest.java +++ b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLLexerTest.java @@ -138,6 +138,11 @@ void testPreproc_Region() { assertMatch("#Region EndIf", BSLLexer.HASH, BSLLexer.PREPROC_REGION, BSLLexer.PREPROC_IDENTIFIER); } + @Test + void testDate() { + assertMatch(BSLLexer.DEFAULT_TOKEN_CHANNEL,"'000dg10101'", BSLLexer.DATETIME, BSLLexer.DATETIME, BSLLexer.DATETIME, BSLLexer.DATETIME); + assertMatch(BSLLexer.DEFAULT_TOKEN_CHANNEL,"'00010101010101'", BSLLexer.DATETIME, BSLLexer.DATETIME, BSLLexer.DATETIME); + } @Test void testString() { diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java index e7a014f8..ad693678 100644 --- a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java +++ b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java @@ -35,7 +35,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.*; class BSLParserTest { private BSLParser parser = new BSLParser(null); @@ -565,6 +565,11 @@ void testDefaultValue() { setInput("ИСТИНА"); assertMatches(parser.constValue()); + + setInput("'000dg10101'"); + assertMatches(parser.constValue()); + setInput("'000dg1010sdaff a sffalgksg1'"); + assertEquals("'00010101'", parser.constValue().getText()); } @Test From a786167636e027e2ddbda53dcb7efbf0b0d7773a Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Mon, 23 Dec 2019 12:13:41 +0300 Subject: [PATCH 6/9] Revert parser to original Signed-off-by: Evgeniy --- src/main/antlr/BSLParser.g4 | 152 ++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 77 deletions(-) diff --git a/src/main/antlr/BSLParser.g4 b/src/main/antlr/BSLParser.g4 index 4982443b..514382a8 100644 --- a/src/main/antlr/BSLParser.g4 +++ b/src/main/antlr/BSLParser.g4 @@ -45,12 +45,15 @@ preproc_else : PREPROC_ELSE_KEYWORD; preproc_endif : PREPROC_ENDIF_KEYWORD; preproc_expression - : PREPROC_LPAREN preproc_expression PREPROC_RPAREN - | PREPROC_NOT_KEYWORD preproc_expression - | preproc_expression preproc_boolOperation preproc_expression - | preproc_symbol + : ( PREPROC_NOT_KEYWORD? (PREPROC_LPAREN preproc_expression PREPROC_RPAREN ) ) + | preproc_logicalExpression ; - +preproc_logicalOperand + : (PREPROC_LPAREN PREPROC_NOT_KEYWORD? preproc_logicalOperand PREPROC_RPAREN) + | ( PREPROC_NOT_KEYWORD? preproc_symbol ) + ; +preproc_logicalExpression + : preproc_logicalOperand (preproc_boolOperation preproc_logicalOperand)*; preproc_symbol : PREPROC_CLIENT_SYMBOL | PREPROC_ATCLIENT_SYMBOL @@ -142,9 +145,8 @@ subs : sub+; sub : procedure | function; procedure : procDeclaration subCodeBlock ENDPROCEDURE_KEYWORD; function : funcDeclaration subCodeBlock ENDFUNCTION_KEYWORD; -procDeclaration : beforeDeclaration PROCEDURE_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; -funcDeclaration : beforeDeclaration FUNCTION_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; -beforeDeclaration: (preprocessor | compilerDirective | annotation)*; +procDeclaration : (preprocessor | compilerDirective | annotation)* PROCEDURE_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; +funcDeclaration : (preprocessor | compilerDirective | annotation)* FUNCTION_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; subCodeBlock : subVars? codeBlock; // statements @@ -163,27 +165,33 @@ elsifBranch elseBranch : ELSE_KEYWORD codeBlock ; -whileStatement : WHILE_KEYWORD cycleBody; -forStatement : FOR_KEYWORD (forVarStatement|forEachStatement) cycleBody; -forVarStatement : IDENTIFIER ASSIGN expression TO_KEYWORD; -forEachStatement : EACH_KEYWORD IDENTIFIER IN_KEYWORD; -cycleBody : expression DO_KEYWORD codeBlock ENDDO_KEYWORD; - +whileStatement : WHILE_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; +forStatement : FOR_KEYWORD IDENTIFIER ASSIGN expression TO_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; +forEachStatement : FOR_KEYWORD EACH_KEYWORD IDENTIFIER IN_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; tryStatement : TRY_KEYWORD tryCodeBlock EXCEPT_KEYWORD exceptCodeBlock ENDTRY_KEYWORD; returnStatement : RETURN_KEYWORD expression?; executeStatement : EXECUTE_KEYWORD (doCall | callParamList); -labelRef : TILDA labelName=IDENTIFIER; -label : labelRef COLON; -gotoStatement : GOTO_KEYWORD labelRef; +callStatement : ((IDENTIFIER | globalMethodCall) modifier* accessCall) | globalMethodCall; + +labelName : IDENTIFIER; +label : TILDA labelName COLON; +gotoStatement : GOTO_KEYWORD TILDA labelName; tryCodeBlock : codeBlock; exceptCodeBlock : codeBlock; +event + : expression + ; + +handler + : expression + ; addHandlerStatement - : ADDHANDLER_KEYWORD event=expression COMMA handler=expression + : ADDHANDLER_KEYWORD event COMMA handler ; removeHandlerStatement - : REMOVEHANDLER_KEYWORD event=expression COMMA handler=expression + : REMOVEHANDLER_KEYWORD event COMMA handler ; ternaryOperator : QUESTION LPAREN expression COMMA expression COMMA expression RPAREN; @@ -195,70 +203,60 @@ fileCodeBlockBeforeSub fileCodeBlock : codeBlock ; -codeBlock : statement*; +codeBlock : (statement | preprocessor)*; numeric : FLOAT | DECIMAL; paramList : param (COMMA param)*; -param : VAL_KEYWORD? IDENTIFIER (ASSIGN constValue)?; -constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | datetime=DATETIME+; +param : VAL_KEYWORD? IDENTIFIER (ASSIGN defaultValue)?; +defaultValue : constValue; +constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | DATETIME; multilineString : STRINGSTART (STRINGPART | BAR)* STRINGTAIL; string : (STRING | multilineString)+; statement - : preprocessor - | label statement - | assignment - | ifStatement - | whileStatement - | forStatement - | tryStatement - | returnStatement - | continueStatement - | breakStatement - | raiseStatement - | executeStatement - | gotoStatement - | addHandlerStatement - | removeHandlerStatement - | methodCall - | compositionCall - | SEMICOLON + : ( + ( + ( label (callStatement | compoundStatement | assignment | preprocessor)?) + | + (callStatement | compoundStatement | assignment| preprocessor) + ) + SEMICOLON? + ) + | SEMICOLON ; -assignment : lValue preprocessor* ASSIGN expression; +assignment : lValue preprocessor* ASSIGN (preprocessor* expression)?; callParamList : callParam (COMMA callParam)*; callParam : expression?; -expression : preprocessor expression - | expression preprocessor - | IDENTIFIER - | methodCall - | ternaryOperator - | constValue - | newExpression - | LPAREN expression RPAREN - | composition - | unaryMathOperation expression - | expression numberoperation expression - | expression mathOperation expression - | NOT_KEYWORD expression - | expression compareOperation expression - | expression AND_KEYWORD expression - | expression OR_KEYWORD expression - | preprocessor - ; - -numberoperation : MUL | QUOTIENT | MODULO; -compareOperation : LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL; -unaryMathOperation : PLUS | MINUS; -mathOperation : PLUS | MINUS; -newExpression : NEW_KEYWORD (typeName doCall? | doCall); -typeName : IDENTIFIER; -methodCall : methodName=IDENTIFIER doCall; -composition : (IDENTIFIER | methodCall | newExpression|ternaryOperator| LPAREN composition RPAREN) member*; -compositionCall : composition DOT methodCall; -member : DOT accessProperty - | DOT methodCall - | accessIndex; +expression : member (preprocessor* operation preprocessor* member)*; +operation : PLUS | MINUS | MUL | QUOTIENT | MODULO | boolOperation | compareOperation; +compareOperation : LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL; +boolOperation : OR_KEYWORD | AND_KEYWORD; +unaryModifier : NOT_KEYWORD | MINUS | PLUS; +member : unaryModifier? (constValue | complexIdentifier | ( LPAREN expression RPAREN ) modifier*); +newExpression : NEW_KEYWORD typeName doCall? | NEW_KEYWORD doCall; +typeName : IDENTIFIER; +methodCall : methodName doCall; +globalMethodCall : methodName doCall; +methodName : IDENTIFIER; +complexIdentifier: (IDENTIFIER | newExpression | ternaryOperator | globalMethodCall) modifier*; +modifier : accessProperty | accessIndex| accessCall; +acceptor : modifier* (accessProperty | accessIndex); +lValue : (IDENTIFIER | globalMethodCall) acceptor?; +accessCall : DOT methodCall; +accessIndex : LBRACK expression RBRACK; +accessProperty : DOT IDENTIFIER; +doCall : LPAREN callParamList RPAREN; -lValue : IDENTIFIER - | composition (DOT accessProperty|accessIndex); -accessIndex : LBRACK expression RBRACK; -accessProperty : IDENTIFIER; -doCall : LPAREN callParamList RPAREN; +compoundStatement + : ifStatement + | whileStatement + | forStatement + | forEachStatement + | tryStatement + | returnStatement + | continueStatement + | breakStatement + | raiseStatement + | executeStatement + | gotoStatement + | addHandlerStatement + | removeHandlerStatement + ; \ No newline at end of file From 6c41e6360c179316ae0fd5a10c60a1d53d17ff48 Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Mon, 23 Dec 2019 12:14:10 +0300 Subject: [PATCH 7/9] Revert parser to modify state Signed-off-by: Evgeniy --- src/main/antlr/BSLParser.g4 | 152 ++++++++++++++++++------------------ 1 file changed, 77 insertions(+), 75 deletions(-) diff --git a/src/main/antlr/BSLParser.g4 b/src/main/antlr/BSLParser.g4 index 514382a8..4982443b 100644 --- a/src/main/antlr/BSLParser.g4 +++ b/src/main/antlr/BSLParser.g4 @@ -45,15 +45,12 @@ preproc_else : PREPROC_ELSE_KEYWORD; preproc_endif : PREPROC_ENDIF_KEYWORD; preproc_expression - : ( PREPROC_NOT_KEYWORD? (PREPROC_LPAREN preproc_expression PREPROC_RPAREN ) ) - | preproc_logicalExpression + : PREPROC_LPAREN preproc_expression PREPROC_RPAREN + | PREPROC_NOT_KEYWORD preproc_expression + | preproc_expression preproc_boolOperation preproc_expression + | preproc_symbol ; -preproc_logicalOperand - : (PREPROC_LPAREN PREPROC_NOT_KEYWORD? preproc_logicalOperand PREPROC_RPAREN) - | ( PREPROC_NOT_KEYWORD? preproc_symbol ) - ; -preproc_logicalExpression - : preproc_logicalOperand (preproc_boolOperation preproc_logicalOperand)*; + preproc_symbol : PREPROC_CLIENT_SYMBOL | PREPROC_ATCLIENT_SYMBOL @@ -145,8 +142,9 @@ subs : sub+; sub : procedure | function; procedure : procDeclaration subCodeBlock ENDPROCEDURE_KEYWORD; function : funcDeclaration subCodeBlock ENDFUNCTION_KEYWORD; -procDeclaration : (preprocessor | compilerDirective | annotation)* PROCEDURE_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; -funcDeclaration : (preprocessor | compilerDirective | annotation)* FUNCTION_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; +procDeclaration : beforeDeclaration PROCEDURE_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; +funcDeclaration : beforeDeclaration FUNCTION_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; +beforeDeclaration: (preprocessor | compilerDirective | annotation)*; subCodeBlock : subVars? codeBlock; // statements @@ -165,33 +163,27 @@ elsifBranch elseBranch : ELSE_KEYWORD codeBlock ; -whileStatement : WHILE_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; -forStatement : FOR_KEYWORD IDENTIFIER ASSIGN expression TO_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; -forEachStatement : FOR_KEYWORD EACH_KEYWORD IDENTIFIER IN_KEYWORD expression DO_KEYWORD codeBlock ENDDO_KEYWORD; +whileStatement : WHILE_KEYWORD cycleBody; +forStatement : FOR_KEYWORD (forVarStatement|forEachStatement) cycleBody; +forVarStatement : IDENTIFIER ASSIGN expression TO_KEYWORD; +forEachStatement : EACH_KEYWORD IDENTIFIER IN_KEYWORD; +cycleBody : expression DO_KEYWORD codeBlock ENDDO_KEYWORD; + tryStatement : TRY_KEYWORD tryCodeBlock EXCEPT_KEYWORD exceptCodeBlock ENDTRY_KEYWORD; returnStatement : RETURN_KEYWORD expression?; executeStatement : EXECUTE_KEYWORD (doCall | callParamList); -callStatement : ((IDENTIFIER | globalMethodCall) modifier* accessCall) | globalMethodCall; - -labelName : IDENTIFIER; -label : TILDA labelName COLON; -gotoStatement : GOTO_KEYWORD TILDA labelName; +labelRef : TILDA labelName=IDENTIFIER; +label : labelRef COLON; +gotoStatement : GOTO_KEYWORD labelRef; tryCodeBlock : codeBlock; exceptCodeBlock : codeBlock; -event - : expression - ; - -handler - : expression - ; addHandlerStatement - : ADDHANDLER_KEYWORD event COMMA handler + : ADDHANDLER_KEYWORD event=expression COMMA handler=expression ; removeHandlerStatement - : REMOVEHANDLER_KEYWORD event COMMA handler + : REMOVEHANDLER_KEYWORD event=expression COMMA handler=expression ; ternaryOperator : QUESTION LPAREN expression COMMA expression COMMA expression RPAREN; @@ -203,60 +195,70 @@ fileCodeBlockBeforeSub fileCodeBlock : codeBlock ; -codeBlock : (statement | preprocessor)*; +codeBlock : statement*; numeric : FLOAT | DECIMAL; paramList : param (COMMA param)*; -param : VAL_KEYWORD? IDENTIFIER (ASSIGN defaultValue)?; -defaultValue : constValue; -constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | DATETIME; +param : VAL_KEYWORD? IDENTIFIER (ASSIGN constValue)?; +constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | datetime=DATETIME+; multilineString : STRINGSTART (STRINGPART | BAR)* STRINGTAIL; string : (STRING | multilineString)+; statement - : ( - ( - ( label (callStatement | compoundStatement | assignment | preprocessor)?) - | - (callStatement | compoundStatement | assignment| preprocessor) - ) - SEMICOLON? - ) - | SEMICOLON + : preprocessor + | label statement + | assignment + | ifStatement + | whileStatement + | forStatement + | tryStatement + | returnStatement + | continueStatement + | breakStatement + | raiseStatement + | executeStatement + | gotoStatement + | addHandlerStatement + | removeHandlerStatement + | methodCall + | compositionCall + | SEMICOLON ; -assignment : lValue preprocessor* ASSIGN (preprocessor* expression)?; +assignment : lValue preprocessor* ASSIGN expression; callParamList : callParam (COMMA callParam)*; callParam : expression?; -expression : member (preprocessor* operation preprocessor* member)*; -operation : PLUS | MINUS | MUL | QUOTIENT | MODULO | boolOperation | compareOperation; -compareOperation : LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL; -boolOperation : OR_KEYWORD | AND_KEYWORD; -unaryModifier : NOT_KEYWORD | MINUS | PLUS; -member : unaryModifier? (constValue | complexIdentifier | ( LPAREN expression RPAREN ) modifier*); -newExpression : NEW_KEYWORD typeName doCall? | NEW_KEYWORD doCall; -typeName : IDENTIFIER; -methodCall : methodName doCall; -globalMethodCall : methodName doCall; -methodName : IDENTIFIER; -complexIdentifier: (IDENTIFIER | newExpression | ternaryOperator | globalMethodCall) modifier*; -modifier : accessProperty | accessIndex| accessCall; -acceptor : modifier* (accessProperty | accessIndex); -lValue : (IDENTIFIER | globalMethodCall) acceptor?; -accessCall : DOT methodCall; -accessIndex : LBRACK expression RBRACK; -accessProperty : DOT IDENTIFIER; -doCall : LPAREN callParamList RPAREN; +expression : preprocessor expression + | expression preprocessor + | IDENTIFIER + | methodCall + | ternaryOperator + | constValue + | newExpression + | LPAREN expression RPAREN + | composition + | unaryMathOperation expression + | expression numberoperation expression + | expression mathOperation expression + | NOT_KEYWORD expression + | expression compareOperation expression + | expression AND_KEYWORD expression + | expression OR_KEYWORD expression + | preprocessor + ; + +numberoperation : MUL | QUOTIENT | MODULO; +compareOperation : LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL; +unaryMathOperation : PLUS | MINUS; +mathOperation : PLUS | MINUS; +newExpression : NEW_KEYWORD (typeName doCall? | doCall); +typeName : IDENTIFIER; +methodCall : methodName=IDENTIFIER doCall; +composition : (IDENTIFIER | methodCall | newExpression|ternaryOperator| LPAREN composition RPAREN) member*; +compositionCall : composition DOT methodCall; +member : DOT accessProperty + | DOT methodCall + | accessIndex; -compoundStatement - : ifStatement - | whileStatement - | forStatement - | forEachStatement - | tryStatement - | returnStatement - | continueStatement - | breakStatement - | raiseStatement - | executeStatement - | gotoStatement - | addHandlerStatement - | removeHandlerStatement - ; \ No newline at end of file +lValue : IDENTIFIER + | composition (DOT accessProperty|accessIndex); +accessIndex : LBRACK expression RBRACK; +accessProperty : IDENTIFIER; +doCall : LPAREN callParamList RPAREN; From 928990572a1cdafd226a50338a1111deab8098f5 Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Mon, 23 Dec 2019 12:25:54 +0300 Subject: [PATCH 8/9] Remove BSL Extended parser Signed-off-by: Evgeniy --- .../bsl/parser/BSLExtendedParser.java | 83 ------------------- 1 file changed, 83 deletions(-) delete mode 100644 src/main/java/com/github/_1c_syntax/bsl/parser/BSLExtendedParser.java diff --git a/src/main/java/com/github/_1c_syntax/bsl/parser/BSLExtendedParser.java b/src/main/java/com/github/_1c_syntax/bsl/parser/BSLExtendedParser.java deleted file mode 100644 index b36abf37..00000000 --- a/src/main/java/com/github/_1c_syntax/bsl/parser/BSLExtendedParser.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * This file is a part of BSL Parser. - * - * Copyright © 2018-2019 - * Alexey Sosnoviy , Nikita Gryzlov , Sergey Batanov - * - * SPDX-License-Identifier: LGPL-3.0-or-later - * - * BSL Parser is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3.0 of the License, or (at your option) any later version. - * - * BSL Parser is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BSL Parser. - */ -package com.github._1c_syntax.bsl.parser; - -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.atn.PredictionMode; - -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; - -public class BSLExtendedParser extends BSLParser { - - CommonTokenStream tokenStream; - private BSLLexer lexer = new BSLLexer(null); - - public BSLExtendedParser() { - super(null); - } - - public BSLExtendedParser(TokenStream input) { - super(input); - } - - public BSLParser.FileContext parseFile(Path path) { - prepareParser(path); - return file(); - } - - public BSLParser.FileContext parseFile(File file) { - prepareParser(file.toPath()); - try { - this.getInterpreter().setPredictionMode(PredictionMode.SLL); - return file(); - } catch (Exception ex) { - tokenStream.seek(0); // rewind input stream - this.reset(); - this.getInterpreter().setPredictionMode(PredictionMode.LL); - } - - return file(); - } - - private void prepareParser(Path path) { - this.removeErrorListener(ConsoleErrorListener.INSTANCE); - CharStream input; - - try (InputStream fis = new BufferedInputStream(new FileInputStream(path.toAbsolutePath().toString())); - UnicodeBOMInputStream ubis = new UnicodeBOMInputStream(fis) - ) { - - ubis.skipBOM(); - - input = new CaseChangingCharStream(CharStreams.fromStream(ubis), true); - } catch (IOException e) { - throw new RuntimeException(e); - } - - lexer.setInputStream(input); - - tokenStream = new CommonTokenStream(lexer); - this.setInputStream(tokenStream); - } -} From 5d53f170478f5a176c683f7e06841e3bcb048711 Mon Sep 17 00:00:00 2001 From: Evgeniy Date: Tue, 24 Dec 2019 09:38:14 +0300 Subject: [PATCH 9/9] Cleanup grammar Signed-off-by: Evgeniy --- .../bsl/parser/JMXBSLLexerTest.java | 13 +- src/main/antlr/BSLLexer.g4 | 22 +- src/main/antlr/BSLParser.g4 | 372 ++++++++---------- .../_1c_syntax/bsl/parser/BSLParserTest.java | 215 +++++----- .../_1c_syntax/bsl/parser/TokenizerTest.java | 2 +- 5 files changed, 272 insertions(+), 352 deletions(-) diff --git a/src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java b/src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java index 12dd0924..564ce258 100644 --- a/src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java +++ b/src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java @@ -40,7 +40,7 @@ @BenchmarkMode(Mode.AverageTime) @Warmup(iterations = 3) -@Measurement(iterations = 5) +@Measurement(iterations = 5, time=2) @OutputTimeUnit(TimeUnit.MILLISECONDS) public class JMXBSLLexerTest { @@ -85,7 +85,7 @@ public static class MyState { @Param({"As stream", "As text"}) public String mode; public Lexer lexer; - private String fileName = "Module.bsl"; + private String fileName = "LargeModule.bsl"; private String content; public String getContent() { @@ -94,10 +94,15 @@ public String getContent() { @Setup public void init() throws Exception { - Class lexerClass = (Class) Class.forName("com.github._1c_syntax.bsl.parser." + lexerClassName); + Class lexerClass = Class.forName("com.github._1c_syntax.bsl.parser." + lexerClassName); lexer = (Lexer) lexerClass.getDeclaredConstructors()[0].newInstance((Object) null); final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - content = IOUtils.toString(new BufferedInputStream(classLoader.getResourceAsStream(fileName)), Charset.defaultCharset()); + InputStream resourceAsStream = classLoader.getResourceAsStream(fileName); + if(resourceAsStream!=null) { + content = IOUtils.toString(new BufferedInputStream(resourceAsStream), Charset.defaultCharset()); + }else { + throw new RuntimeException("Empty file"); + } } } } diff --git a/src/main/antlr/BSLLexer.g4 b/src/main/antlr/BSLLexer.g4 index e865703f..24b7feec 100644 --- a/src/main/antlr/BSLLexer.g4 +++ b/src/main/antlr/BSLLexer.g4 @@ -67,14 +67,13 @@ BAR: '|'; TILDA: '~' -> pushMode(LABEL_MODE); // literals -TRUE : 'ИСТИНА' | 'TRUE'; -FALSE : 'ЛОЖЬ' | 'FALSE'; -UNDEFINED : 'НЕОПРЕДЕЛЕНО' | 'UNDEFINED'; -NULL : 'NULL'; -DECIMAL: DIGIT+; -DATETIME: QUOTE -> pushMode(DATE_MODE); - -QUOTE: '\''; +TRUE : 'ИСТИНА' | 'TRUE'; +FALSE : 'ЛОЖЬ' | 'FALSE'; +UNDEFINED : 'НЕОПРЕДЕЛЕНО' | 'UNDEFINED'; +NULL : 'NULL'; +DECIMAL : DIGIT+; +DATETIME : SQUOTE(~['\n\r])*SQUOTE?; +SQUOTE : '\''; FLOAT : DIGIT+ '.' DIGIT*; STRING: '"' (~[\r\n"] | '""')* '"'; @@ -342,11 +341,4 @@ DOT_WHITE_SPACE ; DOT_IDENTIFIER : LETTER ( LETTER | DIGIT )* -> type(IDENTIFIER), popMode; -mode DATE_MODE; -DATE_WHITE_SPACE - : ~[\p{Nd}'] - -> channel(HIDDEN) - ; -QUOTE_DATE_MODE: QUOTE -> type(DATETIME), popMode; -DATE_PART : [\p{Nd}]+ -> type(DATETIME); diff --git a/src/main/antlr/BSLParser.g4 b/src/main/antlr/BSLParser.g4 index 4982443b..a2a232b8 100644 --- a/src/main/antlr/BSLParser.g4 +++ b/src/main/antlr/BSLParser.g4 @@ -27,238 +27,184 @@ options { } // ROOT -file: shebang? preprocessor* moduleVars? preprocessor* (fileCodeBlockBeforeSub subs)? fileCodeBlock EOF; +file : shebang? preprocessor* moduleVarBlock? preprocessor* fileCodeBlockBeforeSub=codeBlock? subDeclaration? fileCodeBlock=codeBlock? EOF; // preprocessor -shebang : HASH PREPROC_EXCLAMATION_MARK (PREPROC_ANY | PREPROC_IDENTIFIER)*; - -usedLib : (PREPROC_STRING | PREPROC_IDENTIFIER); -use : PREPROC_USE_KEYWORD usedLib; - -regionStart : PREPROC_REGION regionName; -regionEnd : PREPROC_END_REGION; -regionName : PREPROC_IDENTIFIER; - -preproc_if : PREPROC_IF_KEYWORD preproc_expression PREPROC_THEN_KEYWORD; -preproc_elsif : PREPROC_ELSIF_KEYWORD preproc_expression PREPROC_THEN_KEYWORD; -preproc_else : PREPROC_ELSE_KEYWORD; -preproc_endif : PREPROC_ENDIF_KEYWORD; - -preproc_expression - : PREPROC_LPAREN preproc_expression PREPROC_RPAREN - | PREPROC_NOT_KEYWORD preproc_expression - | preproc_expression preproc_boolOperation preproc_expression - | preproc_symbol - ; - -preproc_symbol - : PREPROC_CLIENT_SYMBOL - | PREPROC_ATCLIENT_SYMBOL - | PREPROC_SERVER_SYMBOL - | PREPROC_ATSERVER_SYMBOL - | PREPROC_MOBILEAPPCLIENT_SYMBOL - | PREPROC_MOBILEAPPSERVER_SYMBOL - | PREPROC_MOBILECLIENT_SYMBOL - | PREPROC_THICKCLIENTORDINARYAPPLICATION_SYMBOL - | PREPROC_THICKCLIENTMANAGEDAPPLICATION_SYMBOL - | PREPROC_EXTERNALCONNECTION_SYMBOL - | PREPROC_THINCLIENT_SYMBOL - | PREPROC_WEBCLIENT_SYMBOL - | preproc_unknownSymbol - ; -preproc_unknownSymbol - : PREPROC_IDENTIFIER - ; -preproc_boolOperation - : PREPROC_OR_KEYWORD - | PREPROC_AND_KEYWORD - ; - -preprocessor - : HASH - (regionStart - | regionEnd - | preproc_if - | preproc_elsif - | preproc_else - | preproc_endif - | use - ) - ; +shebang : HASH PREPROC_EXCLAMATION_MARK (PREPROC_ANY | PREPROC_IDENTIFIER)*; + +usedLib : (PREPROC_STRING | PREPROC_IDENTIFIER); +use : PREPROC_USE_KEYWORD usedLib; + +regionStart : PREPROC_REGION regionName; +regionEnd : PREPROC_END_REGION; +regionName : PREPROC_IDENTIFIER; + +preproc_if : PREPROC_IF_KEYWORD preproc_expression PREPROC_THEN_KEYWORD; +preproc_elsif : PREPROC_ELSIF_KEYWORD preproc_expression PREPROC_THEN_KEYWORD; +preproc_else : PREPROC_ELSE_KEYWORD; +preproc_endif : PREPROC_ENDIF_KEYWORD; + +preproc_expression : PREPROC_LPAREN preproc_expression PREPROC_RPAREN + | PREPROC_NOT_KEYWORD preproc_expression + | preproc_expression PREPROC_AND_KEYWORD preproc_expression + | preproc_expression PREPROC_OR_KEYWORD preproc_expression + | preproc_symbol + ; + +preproc_symbol : PREPROC_CLIENT_SYMBOL + | PREPROC_ATCLIENT_SYMBOL + | PREPROC_SERVER_SYMBOL + | PREPROC_ATSERVER_SYMBOL + | PREPROC_MOBILEAPPCLIENT_SYMBOL + | PREPROC_MOBILEAPPSERVER_SYMBOL + | PREPROC_MOBILECLIENT_SYMBOL + | PREPROC_THICKCLIENTORDINARYAPPLICATION_SYMBOL + | PREPROC_THICKCLIENTMANAGEDAPPLICATION_SYMBOL + | PREPROC_EXTERNALCONNECTION_SYMBOL + | PREPROC_THINCLIENT_SYMBOL + | PREPROC_WEBCLIENT_SYMBOL + | preproc_unknownSymbol + ; +preproc_unknownSymbol : PREPROC_IDENTIFIER + ; + +preprocessor : HASH + (regionStart + | regionEnd + | preproc_if + | preproc_elsif + | preproc_else + | preproc_endif + | use + ) + ; // compiler directives -compilerDirectiveSymbol - : ANNOTATION_ATSERVERNOCONTEXT_SYMBOL - | ANNOTATION_ATCLIENTATSERVERNOCONTEXT_SYMBOL - | ANNOTATION_ATCLIENTATSERVER_SYMBOL - | ANNOTATION_ATCLIENT_SYMBOL - | ANNOTATION_ATSERVER_SYMBOL - ; - -compilerDirective - : AMPERSAND compilerDirectiveSymbol - ; - +compilerDirectiveSymbol : ANNOTATION_ATSERVERNOCONTEXT_SYMBOL + | ANNOTATION_ATCLIENTATSERVERNOCONTEXT_SYMBOL + | ANNOTATION_ATCLIENTATSERVER_SYMBOL + | ANNOTATION_ATCLIENT_SYMBOL + | ANNOTATION_ATSERVER_SYMBOL + ; + +compilerDirective : AMPERSAND compilerDirectiveSymbol; // annotations -annotationName - : ANNOTATION_CUSTOM_SYMBOL - ; -annotationParamName - : IDENTIFIER - ; -annotation - : AMPERSAND annotationName annotationParams? - ; -annotationParams - : LPAREN - ( - annotationParam - (COMMA annotationParam)* - )? - RPAREN - ; -annotationParam - : (annotationParamName (ASSIGN constValue)?) - | constValue - ; +annotationName : ANNOTATION_CUSTOM_SYMBOL; +annotationParamName : IDENTIFIER; +annotation : AMPERSAND annotationName annotationParams?; +annotationParams : LPAREN (annotationParam (COMMA annotationParam)*)? RPAREN; + +annotationParam : (annotationParamName (ASSIGN literal)?) + | literal; // vars -var_name : IDENTIFIER; +var_name : IDENTIFIER; -moduleVars : moduleVar+; -moduleVar : (preprocessor | compilerDirective | annotation)* VAR_KEYWORD moduleVarsList SEMICOLON?; -moduleVarsList : moduleVarDeclaration (COMMA moduleVarDeclaration)*; -moduleVarDeclaration: var_name EXPORT_KEYWORD?; +variableDeclaration : (preprocessor | compilerDirective | annotation)*; -subVars : subVar+; -subVar : (preprocessor | compilerDirective | annotation)* VAR_KEYWORD subVarsList SEMICOLON?; -subVarsList : subVarDeclaration (COMMA subVarDeclaration)*; -subVarDeclaration: var_name; +moduleVarBlock : moduleVar*; +moduleVar : variableDeclaration VAR_KEYWORD moduleVarsList SEMICOLON?; +moduleVarsList : moduleVarDeclaration (COMMA moduleVarDeclaration)*; +moduleVarDeclaration : var_name EXPORT_KEYWORD?; -// subs -subName : IDENTIFIER; +subVar : variableDeclaration VAR_KEYWORD subVarsList SEMICOLON?; +subVarsList : subVarDeclaration (COMMA subVarDeclaration)*; +subVarDeclaration : var_name; -subs : sub+; -sub : procedure | function; -procedure : procDeclaration subCodeBlock ENDPROCEDURE_KEYWORD; -function : funcDeclaration subCodeBlock ENDFUNCTION_KEYWORD; -procDeclaration : beforeDeclaration PROCEDURE_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; -funcDeclaration : beforeDeclaration FUNCTION_KEYWORD subName LPAREN paramList? RPAREN EXPORT_KEYWORD?; -beforeDeclaration: (preprocessor | compilerDirective | annotation)*; -subCodeBlock : subVars? codeBlock; +// subs +subDeclaration : (procedure | function)*; +procedure : procDeclaration subCodeBlock ENDPROCEDURE_KEYWORD; +function : funcDeclaration subCodeBlock ENDFUNCTION_KEYWORD; +procDeclaration : beforeDeclaration PROCEDURE_KEYWORD subName=IDENTIFIER LPAREN paramList? RPAREN EXPORT_KEYWORD?; +funcDeclaration : beforeDeclaration FUNCTION_KEYWORD subName=IDENTIFIER LPAREN paramList? RPAREN EXPORT_KEYWORD?; +beforeDeclaration : (preprocessor | compilerDirective | annotation)*; +subCodeBlock : subVar* codeBlock; // statements -continueStatement : CONTINUE_KEYWORD; -breakStatement : BREAK_KEYWORD; -raiseStatement : RAISE_KEYWORD expression?; -ifStatement - : ifBranch elsifBranch* elseBranch? ENDIF_KEYWORD - ; -ifBranch - : IF_KEYWORD expression THEN_KEYWORD codeBlock - ; -elsifBranch - : ELSIF_KEYWORD expression THEN_KEYWORD codeBlock - ; -elseBranch - : ELSE_KEYWORD codeBlock - ; -whileStatement : WHILE_KEYWORD cycleBody; -forStatement : FOR_KEYWORD (forVarStatement|forEachStatement) cycleBody; -forVarStatement : IDENTIFIER ASSIGN expression TO_KEYWORD; -forEachStatement : EACH_KEYWORD IDENTIFIER IN_KEYWORD; -cycleBody : expression DO_KEYWORD codeBlock ENDDO_KEYWORD; - -tryStatement : TRY_KEYWORD tryCodeBlock EXCEPT_KEYWORD exceptCodeBlock ENDTRY_KEYWORD; -returnStatement : RETURN_KEYWORD expression?; -executeStatement : EXECUTE_KEYWORD (doCall | callParamList); -labelRef : TILDA labelName=IDENTIFIER; -label : labelRef COLON; -gotoStatement : GOTO_KEYWORD labelRef; - -tryCodeBlock : codeBlock; -exceptCodeBlock : codeBlock; +continueStatement : CONTINUE_KEYWORD; +breakStatement : BREAK_KEYWORD; +raiseStatement : RAISE_KEYWORD expression?; +ifStatement : ifBranch elsifBranch* elseBranch? ENDIF_KEYWORD; +ifBranch : IF_KEYWORD expression THEN_KEYWORD codeBlock; +elsifBranch : ELSIF_KEYWORD expression THEN_KEYWORD codeBlock; +elseBranch : ELSE_KEYWORD codeBlock; +whileStatement : WHILE_KEYWORD cycleBody; +forStatement : FOR_KEYWORD (forVarStatement|forEachStatement) cycleBody; +forVarStatement : IDENTIFIER ASSIGN expression TO_KEYWORD; +forEachStatement : EACH_KEYWORD IDENTIFIER IN_KEYWORD; +cycleBody : expression DO_KEYWORD codeBlock ENDDO_KEYWORD; + +tryStatement : TRY_KEYWORD tryCodeBlock=codeBlock EXCEPT_KEYWORD exceptCodeBlock=codeBlock ENDTRY_KEYWORD; +returnStatement : RETURN_KEYWORD expression?; +executeStatement : EXECUTE_KEYWORD (doCall | argumentList); +labelRef : TILDA labelName=IDENTIFIER; +label : labelRef COLON; +gotoStatement : GOTO_KEYWORD labelRef; + +addHandlerStatement : ADDHANDLER_KEYWORD event=expression COMMA handler=expression; +removeHandlerStatement : REMOVEHANDLER_KEYWORD event=expression COMMA handler=expression; + +ternaryOperator : QUESTION LPAREN expression COMMA expression COMMA expression RPAREN; -addHandlerStatement - : ADDHANDLER_KEYWORD event=expression COMMA handler=expression - ; -removeHandlerStatement - : REMOVEHANDLER_KEYWORD event=expression COMMA handler=expression - ; +// main +codeBlock : statement*; -ternaryOperator : QUESTION LPAREN expression COMMA expression COMMA expression RPAREN; +numeric : FLOAT | DECIMAL; +paramList : param (COMMA param)*; +param : VAL_KEYWORD? IDENTIFIER (ASSIGN literal)?; +literal : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | DATETIME; -// main -fileCodeBlockBeforeSub - : codeBlock - ; -fileCodeBlock - : codeBlock - ; -codeBlock : statement*; -numeric : FLOAT | DECIMAL; -paramList : param (COMMA param)*; -param : VAL_KEYWORD? IDENTIFIER (ASSIGN constValue)?; -constValue : (MINUS | PLUS)? numeric | string | TRUE | FALSE | UNDEFINED | NULL | datetime=DATETIME+; -multilineString : STRINGSTART (STRINGPART | BAR)* STRINGTAIL; -string : (STRING | multilineString)+; +multilineString : STRINGSTART (STRINGPART | BAR)* STRINGTAIL; +string : (STRING | multilineString)+; statement - : preprocessor - | label statement - | assignment - | ifStatement - | whileStatement - | forStatement - | tryStatement - | returnStatement - | continueStatement - | breakStatement - | raiseStatement - | executeStatement - | gotoStatement - | addHandlerStatement - | removeHandlerStatement - | methodCall - | compositionCall - | SEMICOLON - ; -assignment : lValue preprocessor* ASSIGN expression; -callParamList : callParam (COMMA callParam)*; -callParam : expression?; -expression : preprocessor expression - | expression preprocessor - | IDENTIFIER - | methodCall - | ternaryOperator - | constValue - | newExpression - | LPAREN expression RPAREN - | composition - | unaryMathOperation expression - | expression numberoperation expression - | expression mathOperation expression - | NOT_KEYWORD expression - | expression compareOperation expression - | expression AND_KEYWORD expression - | expression OR_KEYWORD expression - | preprocessor - ; - -numberoperation : MUL | QUOTIENT | MODULO; -compareOperation : LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL; -unaryMathOperation : PLUS | MINUS; -mathOperation : PLUS | MINUS; -newExpression : NEW_KEYWORD (typeName doCall? | doCall); -typeName : IDENTIFIER; + : preprocessor + | assignmentStatement + | label statement + | ifStatement + | whileStatement + | forStatement + | tryStatement + | returnStatement + | continueStatement + | breakStatement + | raiseStatement + | executeStatement + | gotoStatement + | addHandlerStatement + | removeHandlerStatement + | callStatement + | SEMICOLON; + +assignmentStatement : lValue preprocessor* ASSIGN expression; +callStatement : methodCall + | expression DOT methodCall; methodCall : methodName=IDENTIFIER doCall; -composition : (IDENTIFIER | methodCall | newExpression|ternaryOperator| LPAREN composition RPAREN) member*; -compositionCall : composition DOT methodCall; -member : DOT accessProperty - | DOT methodCall - | accessIndex; + +expression : LPAREN expression RPAREN + | IDENTIFIER + | methodCall + | ternaryOperator + | literal + | newExpression + | expression DOT expression + | expression LBRACK expression RBRACK + | (PLUS | MINUS) expression + | expression (MUL | QUOTIENT | MODULO) expression + | expression (PLUS | MINUS) expression + | NOT_KEYWORD expression + | expression (LESS | LESS_OR_EQUAL | GREATER | GREATER_OR_EQUAL | ASSIGN | NOT_EQUAL) expression + | expression AND_KEYWORD expression + | expression OR_KEYWORD expression + | preprocessor expression + | expression preprocessor + | preprocessor + ; + +newExpression : NEW_KEYWORD (typeName=IDENTIFIER doCall? | doCall); lValue : IDENTIFIER - | composition (DOT accessProperty|accessIndex); + | expression (DOT accessProperty|accessIndex); accessIndex : LBRACK expression RBRACK; accessProperty : IDENTIFIER; -doCall : LPAREN callParamList RPAREN; +doCall : LPAREN argumentList? RPAREN; +argumentList : expression? (COMMA expression?)*; diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java index ad693678..f7b96c1d 100644 --- a/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java +++ b/src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java @@ -56,7 +56,7 @@ private void setInput(String inputString, int mode) { CharStream inputTemp = CharStreams.fromStream(ubis, StandardCharsets.UTF_8); input = new CaseChangingCharStream(inputTemp, true); - + } catch (IOException e) { throw new RuntimeException(e); } @@ -89,10 +89,10 @@ private void assertMatches(ParseTree tree) throws RecognitionException { boolean parseSuccess = ((BSLLexer) parser.getInputStream().getTokenSource())._hitEOF; if (!parseSuccess) { throw new RecognitionException( - "Parse error EOF don't hit\n" + parser.getInputStream().getText(), - parser, - parser.getInputStream(), - parser.getContext() + "Parse error EOF don't hit\n" + parser.getInputStream().getText(), + parser, + parser.getInputStream(), + parser.getContext() ); } } @@ -115,36 +115,36 @@ void testFile() { assertNotMatches(parser.file()); setInput("Перем А; \n" + - "Перем Б; \n" + - "Сообщить();" + "Перем Б; \n" + + "Сообщить();" ); assertMatches(parser.file()); setInput("Перем А; \n" + - "Перем Б; \n" + - "Процедура В()\n" + - "КонецПроцедуры\n" + - "Сообщить();\n" + "Перем Б; \n" + + "Процедура В()\n" + + "КонецПроцедуры\n" + + "Сообщить();\n" ); assertMatches(parser.file()); setInput("#!\n" + - "#Если Сервер Тогда\n" + - "Перем А; \n" + - "Перем Б; \n" + - "#Область Г\n" + - "Процедура В()\n" + - "КонецПроцедуры\n" + - "#КонецОбласти\n" + - "Сообщить();\n" + - "#КонецЕсли\n" + "#Если Сервер Тогда\n" + + "Перем А; \n" + + "Перем Б; \n" + + "#Область Г\n" + + "Процедура В()\n" + + "КонецПроцедуры\n" + + "#КонецОбласти\n" + + "Сообщить();\n" + + "#КонецЕсли\n" ); assertMatches(parser.file()); } @Test - void testShebang(){ + void testShebang() { setInput("#!"); assertMatches(parser.shebang()); @@ -277,19 +277,7 @@ void testPreproc_symbol() { } - @Test - void TestPreproc_boolOperation() { - - setInput("И", BSLLexer.PREPROCESSOR_MODE); - assertMatches(parser.preproc_boolOperation()); - setInput("ИЛИ", BSLLexer.PREPROCESSOR_MODE); - assertMatches(parser.preproc_boolOperation()); - - setInput("НЕ", BSLLexer.PREPROCESSOR_MODE); - assertNotMatches(parser.preproc_boolOperation()); - - } @Test void TestPreprocessor() { @@ -425,40 +413,40 @@ void testExecuteStatement() { @Test void testComplexIdentifier() { setInput("Запрос.Пустой()"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("Запрос.Выполнить()"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("Запрос. Выполнить()"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("?(Истина, Истина, Ложь).Выполнить()"); assertMatches(parser.expression()); setInput("?(Истина, М, М)[0]"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("?(Истина, С, С).Свойство"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("А"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("А()"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("А.А()"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("А[Б]"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("Новый Массив"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("Новый(\"Файл\").Существует()"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); } @@ -518,58 +506,57 @@ void testAssignment() { "#EndRegion\n" + "0\n" + "#EndRegion"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А = А"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А = А + Б[В]"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А = А + Б[В] * Метод()"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А = (А + Б[В] * Метод()) + Модуль.Метод()"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А = Модуль.Метод().Свойство"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А = Модуль.Метод(А).Свойство[А]"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А = Б = В.Метод(А)"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А.Свойство[0] = В.Метод(А)"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А[0].Свойство = В.Метод(А)"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А.Метод()[0][1].Метод().Свойство = В.Метод(А)"); - assertMatches(parser.assignment()); + assertMatches(parser.assignmentStatement()); setInput("А.Свойство.Метод() = В.Метод(А)"); - assertNotMatches(parser.assignment()); + assertNotMatches(parser.assignmentStatement()); setInput("Модуль.Метод().Свойство[А]"); - assertNotMatches(parser.assignment()); + assertNotMatches(parser.assignmentStatement()); } @Test void testDefaultValue() { setInput("0"); - assertMatches(parser.constValue()); + assertMatches(parser.literal()); setInput("-1"); - assertMatches(parser.constValue()); + assertMatches(parser.literal()); setInput("+1"); - assertMatches(parser.constValue()); + assertMatches(parser.literal()); setInput("ИСТИНА"); - assertMatches(parser.constValue()); + assertMatches(parser.literal()); setInput("'000dg10101'"); - assertMatches(parser.constValue()); - setInput("'000dg1010sdaff a sffalgksg1'"); - assertEquals("'00010101'", parser.constValue().getText()); + assertMatches(parser.literal()); + } @Test @@ -603,20 +590,20 @@ void testExpression() { assertMatches(parser.expression()); setInput("A1 + \n" + - "#Если (Клиент) Тогда\n" + - "А +\n" + - "#КонецЕсли\n" + - "#Если Клиент Тогда\n" + - "Б +\n" + - "#Иначе\n" + - "#Область Имя\n" + - "В(\n" + - "А + \n" + - "Б\n" + - ")\n" + - "#КонецОбласти\n" + - "#КонецЕсли\n" + - "+ С\n"); + "#Если (Клиент) Тогда\n" + + "А +\n" + + "#КонецЕсли\n" + + "#Если Клиент Тогда\n" + + "Б +\n" + + "#Иначе\n" + + "#Область Имя\n" + + "В(\n" + + "А + \n" + + "Б\n" + + ")\n" + + "#КонецОбласти\n" + + "#КонецЕсли\n" + + "+ С\n"); assertMatches(parser.expression()); setInput("Метод()"); @@ -645,20 +632,20 @@ void testExpression() { assertMatches(parser.expression()); setInput("А = Выполнить"); - assertNotMatches(parser.composition()); + assertNotMatches(parser.expression()); } @Test void tesForEach() { setInput("Для каждого Переменная Из Коллекция Цикл\n" + - "\t\n" + - "КонецЦикла"); + "\t\n" + + "КонецЦикла"); assertMatches(parser.forStatement()); setInput("For Each varible In collection Do\n" + - "\n" + - "EndDo"); + "\n" + + "EndDo"); assertMatches(parser.forStatement()); } @@ -761,7 +748,7 @@ void TestDoCall() { void TestAccessProperty() { setInput(".А"); - assertMatches(parser.member()); + assertMatches(parser.accessProperty()); setInput("А.А"); assertNotMatches(parser.accessProperty()); @@ -783,7 +770,7 @@ void TestAccessIndex() { void TestAccessCall() { setInput(".А(А)"); - assertMatches(parser.member()); + assertMatches(parser.expression()); } @@ -791,27 +778,16 @@ void TestAccessCall() { void TestModifier() { setInput("[А]"); - assertMatches(parser.member()); + assertMatches(parser.expression()); setInput(".А"); - assertMatches(parser.member()); + assertMatches(parser.expression()); setInput(".А(А)"); - assertMatches(parser.member()); + assertMatches(parser.expression()); setInput("А[A]"); - assertNotMatches(parser.member()); - - } - - @Test - void TestTypeName() { - - setInput("Массив"); - assertMatches(parser.typeName()); - - setInput("Выполнить"); - assertNotMatches(parser.typeName()); + assertNotMatches(parser.expression()); } @@ -845,10 +821,10 @@ void TestNewExpression() { void TestMember() { setInput("Истина"); - assertMatches(parser.constValue()); + assertMatches(parser.expression()); setInput("А"); - assertMatches(parser.composition()); + assertMatches(parser.expression()); setInput("(А)"); assertMatches(parser.expression()); @@ -867,14 +843,15 @@ void TestMember() { @Test void TestUnaryModifier() { + /* setInput("-"); assertMatches(parser.unaryMathOperation()); setInput("+"); assertMatches(parser.unaryMathOperation()); - +*/ } - +/* @Test void TestCompareOperation() { @@ -913,25 +890,25 @@ void TestOperation() { setInput("%"); assertMatches(parser.numberoperation()); - } + }*/ @Test void TestCallParam() { setInput(""); - assertMatches(parser.callParam()); + assertMatches(parser.expression()); setInput("А"); - assertMatches(parser.callParam()); + assertMatches(parser.expression()); setInput("НЕ А"); - assertMatches(parser.callParam()); + assertMatches(parser.expression()); setInput("НЕ"); - assertNotMatches(parser.callParam()); + assertNotMatches(parser.expression()); setInput("Если А Тогда"); - assertNotMatches(parser.callParam()); + assertNotMatches(parser.expression()); } @@ -939,13 +916,13 @@ void TestCallParam() { void TestCallParamList() { setInput("НЕ А"); - assertMatches(parser.callParamList()); + assertMatches(parser.expression()); setInput("НЕ А, А"); - assertMatches(parser.callParamList()); + assertMatches(parser.expression()); setInput("НЕ, Если"); - assertNotMatches(parser.callParamList()); + assertNotMatches(parser.expression()); } @@ -983,18 +960,18 @@ void TestCallStatement() { setInput("Сообщить(А, 1)"); assertMatches(parser.methodCall()); setInput("А.А[1].А(А)"); - assertMatches(parser.compositionCall()); + assertMatches(parser.expression()); setInput("А.А()"); - assertMatches(parser.compositionCall()); + assertMatches(parser.expression()); setInput("А.А(А)"); - assertMatches(parser.compositionCall()); + assertMatches(parser.expression()); setInput("А(А).А()"); - assertMatches(parser.compositionCall()); + assertMatches(parser.expression()); setInput("А(А).А.А().А()"); - assertMatches(parser.compositionCall()); + assertMatches(parser.expression()); setInput("ВызватьИсключение А"); - assertNotMatches(parser.compositionCall()); + assertNotMatches(parser.expression()); } @Test diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java index bc99b102..09407f22 100644 --- a/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java +++ b/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java @@ -53,7 +53,7 @@ void computeAST() throws IOException { final BSLParser.FileContext ast = tokenizer.getAst(); // then - BSLParser.FileCodeBlockContext fileCodeBlock = ast.fileCodeBlock(); + BSLParser.CodeBlockContext fileCodeBlock = ast.fileCodeBlock; assertThat(fileCodeBlock).isNotNull(); assertThat(fileCodeBlock.getStart().getType()).isEqualTo(BSLParser.IF_KEYWORD); assertThat(fileCodeBlock.getStop().getType()).isEqualTo(BSLParser.ENDIF_KEYWORD);