Change grammar

src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java

@@ -24,41 +24,85 @@
 import org.antlr.v4.runtime.Lexer;
 import org.apache.commons.io.IOUtils;
 import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.profile.StackProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
 import java.lang.reflect.InvocationTargetException;
-import java.nio.charset.StandardCharsets;
+import java.nio.Buffer;
+import java.nio.charset.Charset;
+import java.util.concurrent.TimeUnit;
 
-@BenchmarkMode(Mode.SampleTime)
-@Warmup(iterations = 2) // число итераций для прогрева нашей функции
-@Measurement(iterations = 2, batchSize = 2)
-@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5, time=2)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
 public class JMXBSLLexerTest {
 
- @Param({"BSLLexer"})
- public String lexerClassName;
 
- private String content;
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .addProfiler(StackProfiler.class)
+ .addProfiler(GCProfiler.class)
+ .build();
 
- public JMXBSLLexerTest() {
- final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- try (InputStream inputStream = classLoader.getResourceAsStream("Module.bsl")) {
- assert inputStream != null;
- content = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
- } catch (IOException e) {
- e.printStackTrace();
+ new Runner(opt).run();
  }
- }
 
- @Benchmark
- public void testCharStream()
- throws ClassNotFoundException, InvocationTargetException, InstantiationException, IllegalAccessException {
+ @Benchmark
+ @Fork(1)
+ public void testLexer(Blackhole blackhole, MyState state) throws IOException {
+ Tokenizer tokenizer;
+ if (state.mode.equals("As stream")) {
+ tokenizer = new Tokenizer(Thread.currentThread().getContextClassLoader().getResourceAsStream(state.fileName), state.lexer);
+ } else {
+ tokenizer = new Tokenizer(state.content, state.lexer);
+ }
+ blackhole.consume(tokenizer.getTokens());
+ }
+
+ @Benchmark
+ @Fork(1)
+ public void testParser(Blackhole blackhole, MyState state) throws IOException {
+ Tokenizer tokenizer;
+ if (state.mode.equals("As stream")) {
+ tokenizer = new Tokenizer(Thread.currentThread().getContextClassLoader().getResourceAsStream(state.fileName), state.lexer);
+ } else {
+ tokenizer = new Tokenizer(state.content, state.lexer);
+ }
+ blackhole.consume(tokenizer.getAst());
+ }
 
- Class<Lexer> lexerClass = (Class<Lexer>) Class.forName("com.github._1c_syntax.bsl.parser." + lexerClassName);
- Lexer lexer = (Lexer) lexerClass.getDeclaredConstructors()[0].newInstance((Object) null);
+ @State(Scope.Thread)
+ public static class MyState {
 
- Tokenizer tokenizer = new Tokenizer(content, lexer);
- tokenizer.getTokens();
- }
+ public String lexerClassName = "BSLLexer";
+ @Param({"As stream", "As text"})
+ public String mode;
+ public Lexer lexer;
+ private String fileName = "LargeModule.bsl";
+ private String content;
+
+ public String getContent() {
+ return content;
+ }
+
+ @Setup
+ public void init() throws Exception {
+ Class<?> lexerClass = Class.forName("com.github._1c_syntax.bsl.parser." + lexerClassName);
+ lexer = (Lexer) lexerClass.getDeclaredConstructors()[0].newInstance((Object) null);
+ final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ InputStream resourceAsStream = classLoader.getResourceAsStream(fileName);
+ if(resourceAsStream!=null) {
+ content = IOUtils.toString(new BufferedInputStream(resourceAsStream), Charset.defaultCharset());
+ }else {
+ throw new RuntimeException("Empty file");
+ }
+ }
+ }
 }

src/main/antlr/BSLLexer.g4

@@ -62,17 +62,18 @@ QUESTION: '?';
 AMPERSAND: '&' -> pushMode(ANNOTATION_MODE);
 HASH: '#' -> pushMode(PREPROCESSOR_MODE);
 
-SQUOTE: '\'';
+
 BAR: '|';
 TILDA: '~' -> pushMode(LABEL_MODE);
 
 // literals
-TRUE : 'ИСТИНА' | 'TRUE';
-FALSE : 'ЛОЖЬ' | 'FALSE';
-UNDEFINED : 'НЕОПРЕДЕЛЕНО' | 'UNDEFINED';
-NULL : 'NULL';
-DECIMAL: DIGIT+;
-DATETIME: SQUOTE(~['\n\r])*SQUOTE?; // TODO: Честная регулярка
+TRUE : 'ИСТИНА' | 'TRUE';
+FALSE : 'ЛОЖЬ' | 'FALSE';
+UNDEFINED : 'НЕОПРЕДЕЛЕНО' | 'UNDEFINED';
+NULL : 'NULL';
+DECIMAL : DIGIT+;
+DATETIME : SQUOTE(~['\n\r])*SQUOTE?;
+SQUOTE : '\'';
 
 FLOAT : DIGIT+ '.' DIGIT*;
 STRING: '"' (~[\r\n"] | '""')* '"';
@@ -339,3 +340,5 @@ DOT_WHITE_SPACE
  type(WHITE_SPACE)
  ;
 DOT_IDENTIFIER : LETTER ( LETTER | DIGIT )* -> type(IDENTIFIER), popMode;
+
+