From 915467a76fe29f6189e57c5cf93251fbbaf446c2 Mon Sep 17 00:00:00 2001 From: Tim Boudreau Date: Mon, 17 Sep 2018 18:17:03 -0400 Subject: [PATCH 1/5] Enable this module to be used in other versions of NetBeans than the one developed against --- .gitignore | 1 + pom.xml | 24 +++++++++--- .../github/drrb/rust/netbeans/Installer.java | 38 +++++++++++++++++++ src/main/nbm/manifest.mf | 1 + 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 src/main/java/com/github/drrb/rust/netbeans/Installer.java diff --git a/.gitignore b/.gitignore index abb9cc7..0aec940 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ Vagrantfile /build /cluster-path +nb-configuration.xml diff --git a/pom.xml b/pom.xml index 04a2a60..9b17fa1 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.github.drrb rust-netbeans - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT nbm Rust NetBeans Plugin @@ -224,13 +224,12 @@ ${netbeans.version} - org.netbeans.api org-netbeans-modules-java-project @@ -320,6 +319,15 @@ guice 3.0 + + + org.netbeans.contrib.yenta + api + 1.1 + @@ -439,16 +447,22 @@ nbm-maven-plugin true + drrb.rust + warn GPL3 LICENSE.txt org.netbeans.modules:org-netbeans-modules-gsf-testrunner - impl + org.netbeans.modules.gsf.testrunner/2 > 1.0 org.netbeans.modules:org-netbeans-modules-gsf-testrunner-ui - impl + org.netbeans.modules.gsf.testrunner.ui > 1.0 + + + org.netbeans.modules:org-netbeans-modules-projectui + loose diff --git a/src/main/java/com/github/drrb/rust/netbeans/Installer.java b/src/main/java/com/github/drrb/rust/netbeans/Installer.java new file mode 100644 index 0000000..3ca06aa --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/Installer.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import org.netbeans.contrib.yenta.Yenta; + +/** + * Allows deployment in multiple versions of NetBeans, with the caveat + * that either of these modules may have broken its ABI in the meantime + * (but in practice, they haven't changed in years). + * + * @author Tim Boudreau + */ +public class Installer extends Yenta { + + @Override + protected Set siblings() { + return new HashSet<>(Arrays.asList("org.netbeans.modules.gsf.testrunner", "org.netbeans.modules.gsf.testrunner.ui")); + } + +} diff --git a/src/main/nbm/manifest.mf b/src/main/nbm/manifest.mf index cb4d374..68dd5c9 100644 --- a/src/main/nbm/manifest.mf +++ b/src/main/nbm/manifest.mf @@ -1,3 +1,4 @@ Manifest-Version: 1.0 OpenIDE-Module-Layer: com/github/drrb/rust/netbeans/layer.xml OpenIDE-Module-Localizing-Bundle: com/github/drrb/rust/netbeans/Bundle.properties +OpenIDE-Module-Install: com/github/drrb/rust/netbeans/Installer.class From 869e6350d56be19728f3a49f8945a6486c178122 Mon Sep 17 00:00:00 2001 From: Tim Boudreau Date: Mon, 1 Oct 2018 13:31:21 -0400 Subject: [PATCH 2/5] Converted to use the Antlr grammar found here - https://github.com/jorendorff/rust-grammar - tests pass, and generally it works. Currently we munge the grammar rules somewhat, to add explicit token names for tokens which were defined inline; this also fixes some issues in the original grammar by introducing a stable order for token definition (for example, not needing to define '<<' as '<' '<'). Next tasks: Move our munging to a fork of the original grammar; address cases from the rust lang sources that do not parse; introduce named sub-rules that allow the parser to easily identify things such as macro and function invocations (which the original grammar just bundles into "expr'), so that features such as find usages are implementable. Some remnants of javacc-related code persist, particularly in tests which were adapted to run using the new parser. --- .gitignore | 3 + .gitmodules | 4 + externals/rust-grammar | 1 + pom.xml | 270 ++- src/main/antlr/imports/rust_tokens.g4 | 121 + .../drrb/rust/netbeans/RustLanguage.java | 34 +- .../drrb/rust/netbeans/cargo/CargoConfig.java | 22 +- .../commandrunner/CommandRunnerUi.java | 2 + .../formatting/RustDocumentFormatter.java | 25 +- .../netbeans/formatting/RustFormatter.java | 5 +- .../highlighting/RustBracesMatcher.java | 88 +- .../RustCompileErrorHighlighter.java | 17 +- .../highlighting/RustSemanticAnalyzer.java | 114 - .../rust/netbeans/indexing/RustIndexer.java | 80 +- .../keypress/RustBreakInterceptor.java | 26 +- .../netbeans/parsing/NetbeansRustLexer.java | 132 -- .../netbeans/parsing/NetbeansRustParser.java | 125 - .../netbeans/parsing/OffsetRustToken.java | 9 +- .../parsing/RustLanguageHierarchy.java | 46 - .../rust/netbeans/parsing/RustTokenId.java | 211 -- .../antlr/AntlrRustLanguageHierarchy.java | 67 + .../AntlrRustLexUtils.java} | 38 +- .../parsing/antlr/AntlrStreamAdapter.java | 162 ++ .../netbeans/parsing/antlr/AntlrTokenID.java | 105 + .../netbeans/parsing/antlr/AntlrTokenIDs.java | 239 ++ .../netbeans/parsing/antlr/AntlrUtils.java | 80 + .../parsing/antlr/CommonRustTokenIDs.java | 122 + .../rust/netbeans/parsing/antlr/ErrImpl.java | 110 + .../netbeans/parsing/antlr/RustAnalyzer.java | 499 ++++ .../parsing/antlr/RustAntlrLexer.java | 160 ++ .../parsing/antlr/RustAntlrParser.java | 90 + .../parsing/antlr/RustAntlrParserResult.java | 87 + .../antlr/RustAntlrSemanticAnalyzer.java | 78 + .../antlr/RustAntlrStructureScanner.java | 61 + .../parsing/antlr/RustElementKind.java | 89 + .../parsing/antlr/RustFoldTypeProvider.java | 55 + .../netbeans/parsing/antlr/RustParseInfo.java | 147 ++ .../parsing/antlr/RustSourceRegion.java | 46 + .../parsing/antlr/RustStructureItem.java | 35 + .../parsing/antlr/RustStructureItemImpl.java | 207 ++ .../parsing/antlr/RustVisibility.java | 52 + .../parsing/antlr/SemanticRegion.java | 191 ++ .../netbeans/parsing/antlr/SyntaxError.java | 147 ++ .../TokenCategorizer.java} | 18 +- .../netbeans/parsing/index/RustStruct.java | 4 + .../parsing/index/RustStructBody.java | 14 +- .../parsing/index/RustStructField.java | 4 + .../parsing/javacc/JavaccCharStream.java | 132 -- .../netbeans/parsing/javacc/RustToken.java | 150 +- .../parsing/javacc/RustTokenFactory.java | 42 - .../parsing/javacc/SimpleCharStream.java | 500 ---- .../rust/netbeans/rustbridge/RustLexer.java | 4 +- .../rust/netbeans/rustbridge/RustToken.java | 20 +- .../drrb/rust/netbeans/FontAndColors.xml | 4 +- .../data/index/project/struct/src/main.rs | 2 +- .../parse/errors/errors_in_items.rs.errors | 110 +- .../errors/missing_parenthesis.rs.errors | 28 +- .../data/parse/errors/two_errors.rs.errors | 55 +- .../blocks/blocks.rs.expected_tokens.json | 156 +- .../block_comment.rs.expected_tokens.json | 26 +- .../comments/comments.rs.expected_tokens.json | 152 +- .../doc_block_comment.rs.expected_tokens.json | 26 +- .../doc_comment.rs.expected_tokens.json | 26 +- ..._doc_block_comment.rs.expected_tokens.json | 26 +- .../inner_doc_comment.rs.expected_tokens.json | 26 +- .../line_comment.rs.expected_tokens.json | 26 +- .../const_items.rs.expected_tokens.json | 106 +- ...ors_in_items.rs.expected_parse_result.json | 37 +- .../errors_in_items.rs.expected_tokens.json | 144 +- ..._parenthesis.rs.expected_parse_result.json | 16 +- ...issing_parenthesis.rs.expected_tokens.json | 34 +- .../two_errors.rs.expected_parse_result.json | 29 +- .../errors/two_errors.rs.expected_tokens.json | 62 +- .../javacc/escapes.rs.expected_tokens.json | 310 ++- ...erator_expressions.rs.expected_tokens.json | 490 ++-- .../binop_assignments.rs.expected_tokens.json | 186 +- .../expressions.rs.expected_tokens.json | 58 +- ...extern_crate_decls.rs.expected_tokens.json | 38 +- .../functions.rs.expected_tokens.json | 76 +- ...ons_and_semicolons.rs.expected_tokens.json | 206 +- .../if_statement.rs.expected_tokens.json | 90 +- .../if_then.rs.expected_tokens.json | 78 +- .../if_then_else.rs.expected_tokens.json | 104 +- ...f_then_elseif_else.rs.expected_tokens.json | 188 +- ...invalid_char.rs.expected_parse_result.json | 8 +- .../invalid_char.rs.expected_tokens.json | 26 +- .../literals/booleans.rs.expected_tokens.json | 58 +- .../literals/bytes.rs.expected_tokens.json | 358 ++- .../literals/numbers.rs.expected_tokens.json | 2008 ++++++++--------- .../literals/strings.rs.expected_tokens.json | 164 +- .../loops/for_loop.rs.expected_tokens.json | 78 +- ...for_loop_enumerate.rs.expected_tokens.json | 102 +- .../for_loop_labels.rs.expected_tokens.json | 186 +- ...alistic_while_loop.rs.expected_tokens.json | 162 +- .../loops/simple_loop.rs.expected_tokens.json | 34 +- .../loops/while_loop.rs.expected_tokens.json | 40 +- .../macro_complex.rs.expected_tokens.json | 138 +- .../macro_simple.rs.expected_tokens.json | 74 +- .../javacc/modules.rs.expected_tokens.json | 130 +- .../ranges/range.rs.expected_tokens.json | 48 +- .../guess.rs.expected_tokens.json | 156 +- .../hello.rs.expected_tokens.json | 36 +- .../javacc/returns.rs.expected_tokens.json | 486 ++-- .../static_items.rs.expected_tokens.json | 104 +- .../javacc/structs.rs.expected_tokens.json | 72 +- .../tests/basic_test.rs.expected_tokens.json | 62 +- .../javacc/typecasts.rs.expected_tokens.json | 54 +- .../unit_expresions.rs.expected_tokens.json | 28 +- .../javacc/use_decls.rs.expected_tokens.json | 310 ++- .../javacc/whitespace.rs.expected_tokens.json | 34 +- src/test/data/semantic/enum.rs.struct | 5 + src/test/data/semantic/impl.rs.struct | 2 + src/test/data/semantic/struct.rs.struct | 5 + src/test/data/semantic/trait.rs.struct | 3 + src/test/data/semantic/trait_impl.rs.struct | 2 + .../com/github/drrb/rust/antlr/TestAntlr.java | 374 +++ .../configuration/RustConfigurationTest.java | 10 +- .../RustSemanticAnalyzerTest.java | 5 - .../RustStructureAnalyzerTest.java | 57 + .../parsing/NetbeansRustLexerTest.java | 54 +- .../parsing/NetbeansRustParserTest.java | 1 - .../parsing/antlr/CommonRustTokenIDsTest.java | 69 + .../parsing/antlr/RustSourcesTest.java | 194 ++ .../netbeans/parsing/javacc/ParseResult.java | 88 +- .../netbeans/parsing/javacc/ParseTest.java | 41 +- .../rust/netbeans/parsing/javacc/TestSrc.java | 83 +- .../parsing/javacc/TokenizationResult.java | 30 +- .../parsing/javacc/TokenizationTest.java | 2 +- .../netbeans/rustbridge/RustLexerTest.java | 108 +- .../rust/netbeans/test/NetbeansWithRust.java | 113 +- .../drrb/rust/netbeans/test/Structure.java | 235 ++ .../rust/netbeans/test/StructureTest.java | 206 ++ .../netbeans/test/TemporaryPreferences.java | 4 + .../modules/csl/api/test/CslTestBase.java | 3 + .../editor/indent/MimeItemFactory.java | 4 +- tmp/.keep | 0 136 files changed, 8852 insertions(+), 5807 deletions(-) create mode 100644 .gitmodules create mode 160000 externals/rust-grammar create mode 100644 src/main/antlr/imports/rust_tokens.g4 delete mode 100644 src/main/java/com/github/drrb/rust/netbeans/highlighting/RustSemanticAnalyzer.java delete mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustLexer.java delete mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustParser.java delete mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/RustLanguageHierarchy.java delete mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/RustTokenId.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLanguageHierarchy.java rename src/main/java/com/github/drrb/rust/netbeans/parsing/{RustLexUtils.java => antlr/AntlrRustLexUtils.java} (69%) create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrStreamAdapter.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenID.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenIDs.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrUtils.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/CommonRustTokenIDs.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/ErrImpl.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAnalyzer.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrLexer.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParser.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParserResult.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrSemanticAnalyzer.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrStructureScanner.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustElementKind.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustFoldTypeProvider.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustParseInfo.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustSourceRegion.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItem.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItemImpl.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustVisibility.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SemanticRegion.java create mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SyntaxError.java rename src/main/java/com/github/drrb/rust/netbeans/parsing/{javacc/ParseUtil.java => antlr/TokenCategorizer.java} (63%) delete mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/JavaccCharStream.java delete mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustTokenFactory.java delete mode 100644 src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/SimpleCharStream.java create mode 100644 src/test/data/semantic/enum.rs.struct create mode 100644 src/test/data/semantic/impl.rs.struct create mode 100644 src/test/data/semantic/struct.rs.struct create mode 100644 src/test/data/semantic/trait.rs.struct create mode 100644 src/test/data/semantic/trait_impl.rs.struct create mode 100644 src/test/java/com/github/drrb/rust/antlr/TestAntlr.java create mode 100644 src/test/java/com/github/drrb/rust/netbeans/highlighting/RustStructureAnalyzerTest.java create mode 100644 src/test/java/com/github/drrb/rust/netbeans/parsing/antlr/CommonRustTokenIDsTest.java create mode 100644 src/test/java/com/github/drrb/rust/netbeans/parsing/antlr/RustSourcesTest.java create mode 100644 src/test/java/com/github/drrb/rust/netbeans/test/Structure.java create mode 100644 src/test/java/com/github/drrb/rust/netbeans/test/StructureTest.java create mode 100644 tmp/.keep diff --git a/.gitignore b/.gitignore index 0aec940..f7ae7ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/tmp/rust/ +/tmp/rust-sources.zip /target/ .not-ci .headless @@ -11,3 +13,4 @@ Vagrantfile /build /cluster-path nb-configuration.xml +/nbproject/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..89628a3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "externals/rust-grammar"] + path = externals/rust-grammar + url = https://github.com/timboudreau/rust-grammar.git + diff --git a/externals/rust-grammar b/externals/rust-grammar new file mode 160000 index 0000000..a88fa08 --- /dev/null +++ b/externals/rust-grammar @@ -0,0 +1 @@ +Subproject commit a88fa087db776ecb6cabcd151ad59c71d128209f diff --git a/pom.xml b/pom.xml index 9b17fa1..3aa611c 100644 --- a/pom.xml +++ b/pom.xml @@ -154,6 +154,12 @@ org-openide-util-ui ${netbeans.version} + + + org.antlr + antlr4-runtime + 4.7.1 + @@ -319,7 +325,7 @@ guice 3.0 - @@ -479,61 +485,218 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - true - -Xlint:unchecked - - - - - org.codehaus.mojo - javacc-maven-plugin - 2.6 + maven-antrun-plugin + 1.8 - generate-parser + download-rust-sources + generate-test-sources + + + + Downloading Rust sources to ${basedir}/tmp/rust-sources.zip (may take a while) + + + + + + + + + + + + + + download-rust-sources + + + + + copy-antlr-grammar generate-sources + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Downloading Rust sources to ${basedir}/tmp/rust-sources.zip if not present + + + + + + + + + + + + + - jjtree-javacc - jjdoc + run - - - net.java.dev.javacc - javacc - 7.0.2 - - - org.codehaus.mojo - exec-maven-plugin - 1.5.0 + org.antlr + antlr4-maven-plugin + 4.7.1 + + ${project.build.directory}/generated-sources/antlr-copied + + ${project.build.directory}/generated-sources/antlr-copied/imports + + true + - map-tokens - generate-sources + antlr - exec + antlr4 + generate-sources - ruby - - src/scripts/generate-rust-token-kind-enum - + true + true + + Java + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + true + -Xlint:unchecked + + @@ -582,13 +745,36 @@ maven-surefire-plugin + java.awt.headless=true: Stop tests hijacking UI on development machine + jna.nosys=true: Don't try to load JNA from the host system, use our dependency instead (needed for Appveyor, which has an old JNA on it) + noverify: Fix for JDK bug with PowerMock + --> -Djava.awt.headless=true -Djna.nosys=true -noverify - 1 + + + + false + 8 false + + + + + false + + + 100 + 100 + 100 + 100 + 100 + true + true + **/*IntegrationTest.java diff --git a/src/main/antlr/imports/rust_tokens.g4 b/src/main/antlr/imports/rust_tokens.g4 new file mode 100644 index 0000000..22f9eb9 --- /dev/null +++ b/src/main/antlr/imports/rust_tokens.g4 @@ -0,0 +1,121 @@ +grammar rust_tokens; +import xidstart, xidcontinue; + +As: 'as'; +Auto: 'auto'; +Box: 'box'; +Break: 'break'; +Crate: 'crate'; +Const: 'const'; +Continue: 'continue'; +Default: 'default'; +Else: 'else'; +Enum: 'enum'; +Extern: 'extern'; +False: 'false'; +For: 'for'; +Let: 'let'; +Loop: 'loop'; +If: 'if'; +Match: 'match'; +Mod: 'mod'; +Move: 'move'; +Pub: 'pub'; +Ref: 'ref'; +Return: 'return'; +Semicolon: ';'; +Super: 'super'; +Static: 'static'; +Struct: 'struct'; +Self: 'self'; +SelfCaps: 'Self'; +True: 'true'; +Type: 'type'; +Trait: 'trait'; +Impl: 'impl'; +Union: 'union'; +Unsafe: 'unsafe'; +Use: 'use'; +Where: 'where'; +While: 'while'; +I32: 'i32'; +I64: 'i64'; +F32: 'f32'; +F64: 'f64'; +Mut: 'mut'; +AttrPrefix: '#['; + +AssignShiftLeft: '<<='; +AssignShiftRight: '>>='; +ByteLiteralPrefix: 'b\''; +ByteStringPrefix: 'b"'; +TripleDot: '...'; +InnerAttrPrefix: '#!['; + +Fn: 'fn'; +In: 'in'; + +HexLiteralPrefix: '0x'; +OctalLiteralPrefix: '0o'; +BitsLiteralPrefix: '0b'; +BlockCommentPrefix: '/*'; +LineCommentPrefix: '//'; + +DoubleAmpersand: '&&'; +DoubleColon: '::'; +DoubleEquals: '=='; +DoublePipe: '||'; +DoubleBackslash: '\\\\'; +FatArrow: '->'; +EqArrow: '=>'; +NotEquals: '!='; +LessThanOrEquals: '<='; +GreaterThanOrEquals: '>='; + +AssignMultiply: '*='; +AssignDivide: '/='; +AssignMod: '%='; +AssignPlus: '+='; +AssignMinus: '-='; +AssignAnd: '&='; +AssignXor: '^='; +AssignOr: '|='; + + +Newline: '\n'; +CarriageReturn: '\r'; + +Ampersand: '&'; +Quote: '"'; +SingleQuote: '\''; +Backslash: '\\'; +Bang: '!'; +Colon: ':'; +Comma: ','; +Dot: '.'; +DotDot: '..'; +LeftParen: '('; +RightParen: ')'; +LeftBrace: '{'; +RightBrace: '}'; +LeftBracket: '['; +RightBracket: ']'; +Equals: '='; +Percent: '%'; +LeftAngleBracket: '<'; +RightAngleBracket: '>'; +Hash: '#'; +Plus: '+'; +Minus: '-'; +Slash: '/'; +Asterisk: '*'; +Underscore: '_'; +QuestionMark: '?'; +AtSymbol: '@'; +Pipe: '|'; +Circumflex: '^'; +Space: ' '; + +Ident: IDENT; + +fragment IDENT: XID_Start XID_Continue*; diff --git a/src/main/java/com/github/drrb/rust/netbeans/RustLanguage.java b/src/main/java/com/github/drrb/rust/netbeans/RustLanguage.java index 100ef8a..3a0ce00 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/RustLanguage.java +++ b/src/main/java/com/github/drrb/rust/netbeans/RustLanguage.java @@ -17,24 +17,27 @@ package com.github.drrb.rust.netbeans; import com.github.drrb.rust.netbeans.formatting.RustFormatter; -import com.github.drrb.rust.netbeans.highlighting.RustSemanticAnalyzer; import com.github.drrb.rust.netbeans.indexing.RustIndexSearcher; import com.github.drrb.rust.netbeans.indexing.RustIndexer; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLanguageHierarchy; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParser; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrSemanticAnalyzer; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrStructureScanner; import org.netbeans.api.lexer.Language; -import org.netbeans.modules.csl.api.Formatter; -import org.netbeans.modules.csl.api.IndexSearcher; import org.netbeans.modules.csl.api.SemanticAnalyzer; import org.netbeans.modules.csl.spi.DefaultLanguageConfig; import org.netbeans.modules.csl.spi.LanguageRegistration; import org.netbeans.modules.parsing.spi.Parser; -import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexerFactory; import org.netbeans.modules.parsing.spi.indexing.PathRecognizerRegistration; import org.openide.util.NbBundle; import java.util.Collections; import java.util.Set; +import org.netbeans.modules.csl.api.Formatter; +import org.netbeans.modules.csl.api.IndexSearcher; +import org.netbeans.modules.csl.api.StructureScanner; +import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexerFactory; @LanguageRegistration(mimeType = RustLanguage.MIME_TYPE) @PathRecognizerRegistration(mimeTypes = RustLanguage.MIME_TYPE, sourcePathIds = RustLanguage.SOURCE_CLASSPATH_ID, libraryPathIds = RustLanguage.BOOT_CLASSPATH_ID, binaryLibraryPathIds = {}) @@ -70,13 +73,13 @@ public boolean isIdentifierChar(char c) { } @Override - public Language getLexerLanguage() { - return RustTokenId.language(); + public Language getLexerLanguage() { + return AntlrRustLanguageHierarchy.INSTANCE.language(); } @Override public Parser getParser() { - return new NetbeansRustParser(); + return new RustAntlrParser(); } @Override @@ -91,19 +94,28 @@ public Formatter getFormatter() { @Override public SemanticAnalyzer getSemanticAnalyzer() { - return new RustSemanticAnalyzer(); + return new RustAntlrSemanticAnalyzer(); + } + + @Override + public StructureScanner getStructureScanner() { + return new RustAntlrStructureScanner(); } @Override public EmbeddingIndexerFactory getIndexerFactory() { return new RustIndexer.Factory(); } - @Override public IndexSearcher getIndexSearcher() { return new RustIndexSearcher(); } + @Override + public boolean hasStructureScanner() { + return true; + } + //TODO: are these required? Is the annotation enough? @Override public Set getLibraryPathIds() { diff --git a/src/main/java/com/github/drrb/rust/netbeans/cargo/CargoConfig.java b/src/main/java/com/github/drrb/rust/netbeans/cargo/CargoConfig.java index 2b46343..6044a9e 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/cargo/CargoConfig.java +++ b/src/main/java/com/github/drrb/rust/netbeans/cargo/CargoConfig.java @@ -16,8 +16,10 @@ */ package com.github.drrb.rust.netbeans.cargo; -import com.github.drrb.rust.netbeans.parsing.RustLexUtils; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLexUtils; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.forLiteralName; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftBrace; import com.github.drrb.rust.netbeans.rustbridge.RustCrateType; import com.github.drrb.rust.netbeans.util.GsfUtilitiesHack; import com.github.drrb.rust.netbeans.util.Template; @@ -26,7 +28,6 @@ import org.netbeans.api.lexer.TokenSequence; import org.openide.filesystems.FileObject; import org.openide.filesystems.FileUtil; -import org.openide.text.NbDocument; import javax.swing.text.Document; import java.io.FileNotFoundException; @@ -69,11 +70,14 @@ private Iterable modFiles(FileObject sourceFile) { @Override public void run() { - TokenSequence rustTokens = new RustLexUtils().getRustTokenSequence(document, 0); + final AntlrTokenID mod = forLiteralName("mod"); + final AntlrTokenID lbrace = leftBrace(); + assert mod != null : "'mod' missing from vocabulary"; + TokenSequence rustTokens = new AntlrRustLexUtils().getRustTokenSequence(document, 0); lookingForModDeclarations: while (rustTokens.moveNext()) { - if (rustTokens.token().id() == RustTokenId.MOD && rustTokens.moveNext()) { - if (rustTokens.moveNext() && rustTokens.token().id() == RustTokenId.LEFT_BRACE) { + if (rustTokens.token().id() == mod && rustTokens.moveNext()) { + if (rustTokens.moveNext() && rustTokens.token().id() == lbrace) { // It's a mod literal continue lookingForModDeclarations; } else { @@ -117,7 +121,11 @@ public List getCrates() { String libCratePath = libCrate.getString("path"); if (libCratePath != null) { FileObject crateFile = baseDir.getFileObject(libCratePath); - List libCrateTypes = new LinkedList<>(libCrate.getList("crate-type")); + List found = libCrate.getList("crate-type"); + List libCrateTypes = new LinkedList<>(); + if (found != null) { + libCrateTypes.addAll(found); + } if (libCrateTypes.isEmpty()) { libCrateTypes.add("rlib"); } diff --git a/src/main/java/com/github/drrb/rust/netbeans/commandrunner/CommandRunnerUi.java b/src/main/java/com/github/drrb/rust/netbeans/commandrunner/CommandRunnerUi.java index 2d2e117..d804b1c 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/commandrunner/CommandRunnerUi.java +++ b/src/main/java/com/github/drrb/rust/netbeans/commandrunner/CommandRunnerUi.java @@ -102,6 +102,8 @@ public void watch(Process process) { process.destroy(); } finally { progressHandle.finish(); + io.getOut().close(); + io.getErr().close(); } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/formatting/RustDocumentFormatter.java b/src/main/java/com/github/drrb/rust/netbeans/formatting/RustDocumentFormatter.java index f55ef85..5801ee9 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/formatting/RustDocumentFormatter.java +++ b/src/main/java/com/github/drrb/rust/netbeans/formatting/RustDocumentFormatter.java @@ -16,8 +16,10 @@ */ package com.github.drrb.rust.netbeans.formatting; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser.NetbeansRustParserResult; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLanguageHierarchy; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParserResult; import org.netbeans.api.lexer.Token; import org.netbeans.api.lexer.TokenHierarchy; import org.netbeans.api.lexer.TokenSequence; @@ -31,6 +33,7 @@ import javax.swing.text.Position; import java.util.LinkedList; import java.util.List; +import org.netbeans.api.lexer.Language; /** * @@ -38,11 +41,11 @@ public class RustDocumentFormatter { private final RustFormatter formatter; - private final NetbeansRustParserResult parseResult; + private final RustAntlrParserResult parseResult; private final BaseDocument document; private final Context context; - RustDocumentFormatter(RustFormatter formatter, NetbeansRustParserResult parseResult, BaseDocument document, Context context) { + RustDocumentFormatter(RustFormatter formatter, RustAntlrParserResult parseResult, BaseDocument document, Context context) { this.formatter = formatter; this.parseResult = parseResult; this.document = document; @@ -51,20 +54,24 @@ public class RustDocumentFormatter { public void format() { final Snapshot snapshot = parseResult.getSnapshot(); + final Language language = AntlrRustLanguageHierarchy.INSTANCE.language(); + final AntlrTokenID leftBrace = CommonRustTokenIDs.leftBrace(); + final AntlrTokenID rightBrace = CommonRustTokenIDs.rightBrace(); + final AntlrTokenID semicolon = CommonRustTokenIDs.semicolon(); try { List delimiters = new LinkedList<>(); TokenHierarchy tokenHierarchy = snapshot.getTokenHierarchy(); - TokenSequence tokenSequence = tokenHierarchy.tokenSequence(RustTokenId.language()); + TokenSequence tokenSequence = tokenHierarchy.tokenSequence(language); tokenSequence.move(0); while (tokenSequence.moveNext()) { - Token token = tokenSequence.token(); + Token token = tokenSequence.token(); int tokenOffset = tokenSequence.offset(); - if (token.id() == RustTokenId.LEFT_BRACE) { + if (token.id() == leftBrace) { delimiters.add(new Delimiter(DelimiterType.OPEN_BRACE, tokenOffset)); - } else if (token.id() == RustTokenId.RIGHT_BRACE) { + } else if (token.id() == rightBrace) { delimiters.add(new Delimiter(DelimiterType.CLOSE_BRACE, tokenOffset)); - } else if (token.id() == RustTokenId.SEMICOLON) { + } else if (token.id() == semicolon) { delimiters.add(new Delimiter(DelimiterType.SEMICOLON, tokenOffset)); } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/formatting/RustFormatter.java b/src/main/java/com/github/drrb/rust/netbeans/formatting/RustFormatter.java index b0e89bc..c612a0e 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/formatting/RustFormatter.java +++ b/src/main/java/com/github/drrb/rust/netbeans/formatting/RustFormatter.java @@ -16,8 +16,7 @@ */ package com.github.drrb.rust.netbeans.formatting; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser.NetbeansRustParserResult; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParserResult; import static java.lang.Character.isWhitespace; import javax.swing.text.BadLocationException; import javax.swing.text.Document; @@ -35,7 +34,7 @@ public class RustFormatter implements Formatter { @Override public void reformat(Context context, ParserResult compilationInfo) { - NetbeansRustParserResult parseResult = (NetbeansRustParser.NetbeansRustParserResult) compilationInfo; + RustAntlrParserResult parseResult = (RustAntlrParserResult) compilationInfo; final BaseDocument document = (BaseDocument) context.document(); final RustDocumentFormatter formatter = new RustDocumentFormatter(this, parseResult, document, context); document.runAtomic(() -> { diff --git a/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustBracesMatcher.java b/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustBracesMatcher.java index 1176fa9..bc75f98 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustBracesMatcher.java +++ b/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustBracesMatcher.java @@ -17,8 +17,14 @@ package com.github.drrb.rust.netbeans.highlighting; import com.github.drrb.rust.netbeans.RustLanguage; -import com.github.drrb.rust.netbeans.parsing.RustLexUtils; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLexUtils; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftAngleBracket; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftBrace; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftBracket; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.rightAngleBracket; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.rightBrace; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.rightBracket; import org.netbeans.api.editor.mimelookup.MimeRegistration; import org.netbeans.api.lexer.Token; import org.netbeans.api.lexer.TokenSequence; @@ -30,6 +36,9 @@ import javax.swing.text.BadLocationException; import java.util.logging.Level; import java.util.logging.Logger; +import org.netbeans.api.lexer.TokenId; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftParen; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.rightParen; /** * @@ -43,7 +52,7 @@ public static class Factory implements BracesMatcherFactory { @Override public BracesMatcher createMatcher(MatcherContext context) { - return new RustBracesMatcher(context, new RustLexUtils()); + return new RustBracesMatcher(context, new AntlrRustLexUtils()); // TODO: is our implementation better than just doing this?: //return BracesMatcherSupport.defaultMatcher(context, -1, -1); // Probably, because it's dealing with tokens instead of characters @@ -51,9 +60,9 @@ public BracesMatcher createMatcher(MatcherContext context) { } } private final MatcherContext context; - private final RustLexUtils rustLexUtils; + private final AntlrRustLexUtils rustLexUtils; - public RustBracesMatcher(MatcherContext context, RustLexUtils rustLexUtils) { + public RustBracesMatcher(MatcherContext context, AntlrRustLexUtils rustLexUtils) { this.context = context; this.rustLexUtils = rustLexUtils; } @@ -64,7 +73,7 @@ public int[] findOrigin() throws InterruptedException, BadLocationException { document.readLock(); try { int offset = context.getSearchOffset(); - TokenSequence tokenSequence = rustLexUtils.getRustTokenSequence(document, offset); + TokenSequence tokenSequence = rustLexUtils.getRustTokenSequence(document, offset); if (tokenSequence == null) { LOGGER.warning("Couldn't get Rust token sequence for braces matching"); return null; @@ -76,12 +85,12 @@ public int[] findOrigin() throws InterruptedException, BadLocationException { } } - private OffsetRange getBraceAtOffset(TokenSequence tokenSequence, int offset) { + private OffsetRange getBraceAtOffset(TokenSequence tokenSequence, int offset) { tokenSequence.move(offset); if (tokenSequence.moveNext()) { - Token token = tokenSequence.token(); - for (BracePair bracePair : BracePair.values()) { - if (token.id() == bracePair.open || token.id() == bracePair.close) { + Token token = tokenSequence.token(); + for (BracePair bracePair : AntlrBracePair.values()) { + if (token.id() == bracePair.open() || token.id() == bracePair.close()) { return OffsetRange.ofCurrentToken(tokenSequence); } } @@ -97,7 +106,7 @@ public int[] findMatches() throws InterruptedException, BadLocationException { document.readLock(); try { int offset = context.getSearchOffset(); - TokenSequence tokenSequence = rustLexUtils.getRustTokenSequence(document, offset); + TokenSequence tokenSequence = rustLexUtils.getRustTokenSequence(document, offset); if (tokenSequence == null) { LOGGER.warning("Couldn't get Rust token sequence for braces matching"); return null; @@ -109,14 +118,14 @@ public int[] findMatches() throws InterruptedException, BadLocationException { } } - private OffsetRange getBraceMatchingTheOneAtOffset(TokenSequence tokenSequence, int offset) { + private OffsetRange getBraceMatchingTheOneAtOffset(TokenSequence tokenSequence, int offset) { tokenSequence.move(offset); if (tokenSequence.moveNext()) { - Token token = tokenSequence.token(); - for (BracePair bracePair : BracePair.values()) { - if (token.id() == bracePair.open) { + Token token = tokenSequence.token(); + for (BracePair bracePair : AntlrBracePair.values()) { + if (token.id() == bracePair.open()) { return findCloseBraceForward(tokenSequence, bracePair); - } else if (token.id() == bracePair.close) { + } else if (token.id() == bracePair.close()) { return findOpenBraceBackward(tokenSequence, bracePair); } } @@ -126,14 +135,14 @@ private OffsetRange getBraceMatchingTheOneAtOffset(TokenSequence to return OffsetRange.NONE; } - private static OffsetRange findCloseBraceForward(TokenSequence tokenSequence, BracePair bracePair) { + private static OffsetRange findCloseBraceForward(TokenSequence tokenSequence, BracePair bracePair) { int balance = 0; while (tokenSequence.moveNext()) { - Token token = tokenSequence.token(); - if (token.id() == bracePair.open) { + Token token = tokenSequence.token(); + if (token.id() == bracePair.open()) { balance++; - } else if (token.id() == bracePair.close) { + } else if (token.id() == bracePair.close()) { if (balance == 0) { return OffsetRange.ofCurrentToken(tokenSequence); } @@ -144,17 +153,17 @@ private static OffsetRange findCloseBraceForward(TokenSequence tokenSequence, BracePair bracePair) { + private static OffsetRange findOpenBraceBackward(TokenSequence tokenSequence, BracePair bracePair) { int balance = 0; while (tokenSequence.movePrevious()) { - Token token = tokenSequence.token(); - if (token.id() == bracePair.open) { + Token token = tokenSequence.token(); + if (token.id() == bracePair.open()) { if (balance == 0) { return OffsetRange.ofCurrentToken(tokenSequence); } balance++; - } else if (token.id() == bracePair.close) { + } else if (token.id() == bracePair.close()) { balance--; } } @@ -162,19 +171,34 @@ private static OffsetRange findOpenBraceBackward(TokenSequence { +public class RustCompileErrorHighlighter extends ParserResultTask { private static final Logger LOG = Logger.getLogger(RustCompileErrorHighlighter.class.getName()); private static final RequestProcessor EXECUTOR = new RequestProcessor("Rust Compile", 12); -// @MimeRegistration(mimeType = RustLanguage.MIME_TYPE, service = TaskFactory.class) + @MimeRegistration(mimeType = RustLanguage.MIME_TYPE, service = TaskFactory.class) public static class Factory extends TaskFactory { @Override @@ -67,14 +69,7 @@ public Collection create(Snapshot snapshot) { } @Override - public void run(NetbeansRustParserResult parseResult, SchedulerEvent event) { - try { - if (parseResult.isFailure()) { - return; - } - } catch (ParseException ex) { - Exceptions.printStackTrace(ex); - } + public void run(RustAntlrParserResult parseResult, SchedulerEvent event) { final Snapshot snapshot = parseResult.getSnapshot(); EXECUTOR.post(new Runnable() { diff --git a/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustSemanticAnalyzer.java b/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustSemanticAnalyzer.java deleted file mode 100644 index ac4755e..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustSemanticAnalyzer.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.highlighting; - -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser.NetbeansRustParserResult; -import com.github.drrb.rust.netbeans.parsing.javacc.*; -import org.netbeans.modules.csl.api.ColoringAttributes; -import org.netbeans.modules.csl.api.OffsetRange; -import org.netbeans.modules.csl.api.SemanticAnalyzer; -import org.netbeans.modules.parsing.spi.ParseException; -import org.netbeans.modules.parsing.spi.Scheduler; -import org.netbeans.modules.parsing.spi.SchedulerEvent; -import org.openide.util.Exceptions; - -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; - -import static com.github.drrb.rust.netbeans.parsing.javacc.ParseUtil.offsetRange; -import static org.netbeans.modules.csl.api.ColoringAttributes.*; - -/** - * - */ -public class RustSemanticAnalyzer extends SemanticAnalyzer { - private static final Logger LOG = Logger.getLogger(RustSemanticAnalyzer.class.getName()); - private final Map> highlights = new HashMap<>(); - private final AtomicBoolean cancelled = new AtomicBoolean(); - - @Override - public void run(NetbeansRustParserResult result, SchedulerEvent event) { - highlights.clear(); - cancelled.set(false); //TODO: respect this cancellation in the visitors - - try { - SimpleNode rootNode = result.rootNode(); - rootNode.jjtAccept(new FunctionNameHighlighter(), null); - rootNode.jjtAccept(new StructHighlighter(), null); - rootNode.jjtAccept(new AnnotationHighlighter(), null); - } catch (ParseException ex) { - Exceptions.printStackTrace(ex); - } - } - - @Override - public Map> getHighlights() { - return new HashMap<>(highlights); - } - - @Override - public int getPriority() { - return 0; - } - - @Override - public Class getSchedulerClass() { - return Scheduler.EDITOR_SENSITIVE_TASK_SCHEDULER; - } - - @Override - public void cancel() { - cancelled.set(true); - } - - private class FunctionNameHighlighter extends RustParserDefaultVisitor { - @Override - public Object visit(ASTfunctionName functionNameNode, Object data) { - RustToken identifier = (RustToken) functionNameNode.jjtGetFirstToken(); - highlights.put(identifier.offsetRange(), EnumSet.of(STATIC, METHOD)); - return null; - } - } - - private class StructHighlighter extends RustParserDefaultVisitor { - @Override - public Object visit(ASTstructName structNameNode, Object data) { - RustToken identifier = (RustToken) structNameNode.jjtGetFirstToken(); - highlights.put(identifier.offsetRange(), CLASS_SET); - return null; - } - - @Override - public Object visit(ASTstructField fieldNode, Object data) { - RustToken identifier = (RustToken) fieldNode.jjtGetFirstToken(); - highlights.put(identifier.offsetRange(), FIELD_SET); - return null; - } - } - - private class AnnotationHighlighter extends RustParserDefaultVisitor { - @Override - public Object visit(ASTAnnotation node, Object data) { - highlights.put(offsetRange(node), EnumSet.of(ANNOTATION_TYPE)); - return null; - } - } -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/indexing/RustIndexer.java b/src/main/java/com/github/drrb/rust/netbeans/indexing/RustIndexer.java index 6c5f35e..7becefe 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/indexing/RustIndexer.java +++ b/src/main/java/com/github/drrb/rust/netbeans/indexing/RustIndexer.java @@ -16,28 +16,24 @@ */ package com.github.drrb.rust.netbeans.indexing; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser.NetbeansRustParserResult; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParserResult; +import com.github.drrb.rust.netbeans.parsing.antlr.RustStructureItem; import com.github.drrb.rust.netbeans.parsing.index.RustStruct; import com.github.drrb.rust.netbeans.parsing.index.RustStructBody; -import com.github.drrb.rust.netbeans.parsing.javacc.ASTStructItem; -import com.github.drrb.rust.netbeans.parsing.javacc.ASTstructName; -import com.github.drrb.rust.netbeans.parsing.javacc.RustParserDefaultVisitor; +import com.github.drrb.rust.netbeans.parsing.index.RustStructField; import org.netbeans.modules.parsing.api.Snapshot; -import org.netbeans.modules.parsing.spi.ParseException; import org.netbeans.modules.parsing.spi.Parser; import org.netbeans.modules.parsing.spi.indexing.Context; import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexer; import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexerFactory; import org.netbeans.modules.parsing.spi.indexing.Indexable; -import org.openide.util.Exceptions; import java.io.IOException; -import java.util.LinkedList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import static com.github.drrb.rust.netbeans.parsing.javacc.ParseUtil.offsetRange; +import java.util.ArrayList; /** * @@ -58,14 +54,47 @@ protected void index(Indexable file, Parser.Result parserResult, Context context LOGGER.log(Level.INFO, "RustIndexer.index({0})", file.getRelativePath()); try { RustIndexWriter indexWriter = index.createIndexWriter(context); - NetbeansRustParserResult parseResult = (NetbeansRustParserResult) parserResult; - IndexingRustVisitor visitor = new IndexingRustVisitor(); - parseResult.rootNode().jjtAccept(visitor, null); - for (RustStruct struct : visitor.structs) { + List structs = findIndexable((RustAntlrParserResult) parserResult); + for (RustStruct struct : structs) { + LOGGER.log(Level.INFO, " include ({0})", struct); indexWriter.write(file, struct); } - } catch (IOException | ParseException ex) { - Exceptions.printStackTrace(ex); + } catch (IOException ex) { + LOGGER.log(Level.WARNING, "Exception indexing " + file.getRelativePath(), ex); + } + } + + private List findIndexable(RustAntlrParserResult parserResult) { + List result = new ArrayList<>(); + for (RustStructureItem structureItem : parserResult.structureItems()) { + RustStruct struct = toRustStruct(structureItem); + if (struct != null) { + result.add(struct); + } + } + return result; + } + + private RustStruct toRustStruct(RustStructureItem item) { + switch (item.rustKind()) { + case ATTR: + case LIFETIME: + case TYPE_REFERENCE: + case TYPE: + return null; + case STRUCT: + RustStruct.Builder structBuilder = RustStruct.builder() + .setOffsetRange(item.range()).setName(item.getName()); + RustStructBody.Builder structBodyBuilder = RustStructBody.builder(); + for (RustStructureItem child : item.getNestedItems()) { + structBodyBuilder.addField(new RustStructField(child.getName(), child.range())); + } + structBuilder.setBody(structBodyBuilder.build()); + return structBuilder.build(); +// case ENUM : +// RustEnum.Builder enumBuilder = RustEnum.builder(); + default: + return null; } } @@ -98,27 +127,4 @@ public int getIndexVersion() { return VERSION; } } - - private static class IndexingRustVisitor extends RustParserDefaultVisitor { - private final List structs = new LinkedList<>(); - - @Override - public Object visit(ASTStructItem structNode, Object data) { - RustStruct.Builder struct = RustStruct.builder() - .setOffsetRange(offsetRange(structNode)); - RustStructBody.Builder structBody = RustStructBody.builder(); - structNode.jjtAccept(new RustParserDefaultVisitor() { - - @Override - public Object visit(ASTstructName node, Object data) { - struct.setName(node.jjtGetFirstToken().image); - return null; - } - - }, null); - struct.setBody(structBody.build()); - structs.add(struct.build()); - return null; - } - } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/keypress/RustBreakInterceptor.java b/src/main/java/com/github/drrb/rust/netbeans/keypress/RustBreakInterceptor.java index 5a4da79..761448e 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/keypress/RustBreakInterceptor.java +++ b/src/main/java/com/github/drrb/rust/netbeans/keypress/RustBreakInterceptor.java @@ -17,8 +17,6 @@ package com.github.drrb.rust.netbeans.keypress; import com.github.drrb.rust.netbeans.RustLanguage; -import com.github.drrb.rust.netbeans.parsing.RustLexUtils; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; import org.netbeans.api.editor.mimelookup.MimePath; import org.netbeans.api.editor.mimelookup.MimeRegistration; import org.netbeans.api.lexer.TokenId; @@ -30,7 +28,9 @@ import javax.swing.text.BadLocationException; import java.util.concurrent.atomic.AtomicBoolean; -import static com.github.drrb.rust.netbeans.parsing.RustTokenId.*; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLexUtils; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; /** * @@ -48,14 +48,16 @@ public boolean beforeInsert(Context context) throws BadLocationException { @Override public void insert(MutableContext ctx) throws BadLocationException { if (cancelled.get()) return; + final AntlrTokenID leftBrace = CommonRustTokenIDs.leftBrace(); + final AntlrTokenID rightBrace = CommonRustTokenIDs.rightBrace(); ContextHolder context = new ContextHolder(ctx); - if (context.previousTokenKind() != LEFT_BRACE) { + if (context.previousTokenKind() != leftBrace) { return; // Only insert a close brace after an open brace } else if (context.nextRowIndent() > context.currentRowIndent()) { return; // There's already stuff in this block - } else if (context.nextTokenKind() == RIGHT_BRACE && context.currentRowIndent() == context.nextRowIndent()) { + } else if (context.nextTokenKind() == rightBrace && context.currentRowIndent() == context.nextRowIndent()) { return; // There's already a closing brace } @@ -74,21 +76,21 @@ public void cancelled(Context context) { private static class ContextHolder { private final MutableContext context; - private final TokenSequence tokenSequence; + private final TokenSequence tokenSequence; private Integer currentRowIndent; private Integer nextRowIndent; private ContextHolder(MutableContext context) { this.context = context; - this.tokenSequence = new RustLexUtils().getRustTokenSequence(context.getDocument(), context.getCaretOffset()); + this.tokenSequence = new AntlrRustLexUtils().getRustTokenSequence(context.getDocument(), context.getCaretOffset()); tokenSequence.move(context.getCaretOffset()); } - private RustTokenId previousTokenKind() { + private TokenId previousTokenKind() { return findNonWhitespaceToken(Direction.BACKWARD); } - private RustTokenId nextTokenKind() { + private TokenId nextTokenKind() { return findNonWhitespaceToken(Direction.FORWARD); } @@ -113,10 +115,10 @@ private int nextRowIndent() throws BadLocationException { return nextRowIndent; } - private RustTokenId findNonWhitespaceToken(Direction direction) { + private TokenId findNonWhitespaceToken(Direction direction) { while (direction.move(tokenSequence)) { - RustTokenId nextTokenKind = tokenSequence.token().id(); - if (nextTokenKind != WHITESPACE) { + TokenId nextTokenKind = tokenSequence.token().id(); + if (nextTokenKind != CommonRustTokenIDs.forSymbolicName("Whitespace")) { return nextTokenKind; } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustLexer.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustLexer.java deleted file mode 100644 index 62ab141..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustLexer.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing; - -import com.github.drrb.rust.netbeans.parsing.javacc.JavaccCharStream; -import com.github.drrb.rust.netbeans.parsing.javacc.RustParserTokenManager; -import com.github.drrb.rust.netbeans.parsing.javacc.TokenMgrError; -import com.github.drrb.rust.netbeans.rustbridge.RustLexer; -import org.netbeans.api.lexer.Token; -import org.netbeans.spi.lexer.Lexer; -import org.netbeans.spi.lexer.LexerInput; -import org.netbeans.spi.lexer.LexerRestartInfo; - -import java.util.Stack; -import java.util.stream.IntStream; - -public class NetbeansRustLexer implements Lexer { - - private static final Token EOF_TOKEN = null; - - private final LexerRestartInfo info; - private final RustParserTokenManager tokenManager; - private final Stack unreturnedTokens = new Stack<>(); - private final LexerInput lexerInput; - private RustLexer lexer = RustLexer.NULL_LEXER; - - public NetbeansRustLexer(LexerRestartInfo info) { - this.info = info; - this.lexerInput = info.input(); - this.tokenManager = new RustParserTokenManager(new JavaccCharStream(info.input())); - } - - @Override - public Token nextToken() { - try { - com.github.drrb.rust.netbeans.parsing.javacc.RustToken token; - if (unreturnedTokens.empty()) { - token = (com.github.drrb.rust.netbeans.parsing.javacc.RustToken) tokenManager.getNextToken(); - log("parsed token: %s%n", token); - while (token.hasSpecialToken()) { - if (token.isEof()) { - info.input().backup(1); - } - log(" token %s has special token %s. pushing %s%n", token, token.specialToken(), token); - unreturnedTokens.push(token); - log(" backing up %s%n", token.image.length()); - lexerInput.backup(token.image.length()); - token = token.specialToken(); - } - } else { - token = unreturnedTokens.pop(); - log("using unreturned token %s%n", token); - IntStream.range(0, token.image.length()).forEach(x -> lexerInput.read()); - } - - log("token = %s%n", token); - log(" read = %s%n", lexerInput.readLength()); - if (token.isEof()) { - return EOF_TOKEN; - } else { - return info.tokenFactory().createToken(RustTokenId.get(token.kind)); - } - } catch (TokenMgrError e) { - return info.tokenFactory().createToken(RustTokenId.GARBAGE); - } - } - - private void log(String format, Object... args) { - //System.out.format(format, args); - } - - private void ensureLexerCreated() { - if (lexer == RustLexer.NULL_LEXER) { - String source = readWholeSource(); - backUp(charsReadThisToken()); - lexer = RustLexer.forString(source); - } - } - - private String readWholeSource() { - reading: - while (readOneCharacter() != LexerInput.EOF) { - continue reading; - } - return charactersReadSoFar(); - } - - protected void backUp(int length) { - info.input().backup(length); - } - - protected int charsReadThisToken() { - return info.input().readLengthEOF(); - } - - protected String charactersReadSoFar() { - return info.input().readText().toString(); - } - - protected int readOneCharacter() { - return info.input().read(); - } - - protected Token createToken(RustTokenId tokenType) { - return info.tokenFactory().createToken(tokenType); - } - - @Override - public Object state() { - return null; - } - - @Override - public void release() { - //lexer.release(); - } - -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustParser.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustParser.java deleted file mode 100644 index 2245628..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustParser.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing; - -import com.github.drrb.rust.netbeans.parsing.javacc.RustParser; -import com.github.drrb.rust.netbeans.parsing.javacc.RustToken; -import com.github.drrb.rust.netbeans.parsing.javacc.SimpleNode; -import org.netbeans.modules.csl.api.Error; -import org.netbeans.modules.csl.api.Severity; -import org.netbeans.modules.csl.spi.DefaultError; -import org.netbeans.modules.csl.spi.ParserResult; -import org.netbeans.modules.parsing.api.Snapshot; -import org.netbeans.modules.parsing.api.Task; -import org.netbeans.modules.parsing.spi.ParseException; -import org.netbeans.modules.parsing.spi.Parser; -import org.netbeans.modules.parsing.spi.SourceModificationEvent; -import org.openide.filesystems.FileObject; - -import javax.swing.event.ChangeListener; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; - -import static java.util.stream.Collectors.toList; - -/** - * - */ -public class NetbeansRustParser extends Parser { - private static final Logger LOG = Logger.getLogger(NetbeansRustParser.class.getName()); - private Snapshot snapshot; - private NetbeansRustParserResult result; - - @Override - public void parse(final Snapshot snapshot, Task task, SourceModificationEvent event) { - this.snapshot = snapshot; - String source = snapshot.getText().toString(); - - RustParser.Result parseResult = RustParser.parse(source); - result = NetbeansRustParserResult.complete(snapshot, parseResult); - } - - @Override - public NetbeansRustParserResult getResult(Task task) throws ParseException { - return result; - } - - @Override - public void addChangeListener(ChangeListener changeListener) { - } - - @Override - public void removeChangeListener(ChangeListener changeListener) { - } - - public static class NetbeansRustParserResult extends ParserResult { - - private final AtomicBoolean valid = new AtomicBoolean(true); - private final List diagnostics; - private final RustParser.Result parseResult; - - public NetbeansRustParserResult(Snapshot snapshot, RustParser.Result parseResult, List diagnostics) { - super(snapshot); - this.parseResult = parseResult; - this.diagnostics = Collections.unmodifiableList(diagnostics); - } - - public SimpleNode rootNode() throws ParseException { - //TODO: i think i've seen the valid field on the parser itself in an example. Where should it be? - //TODO: also, this seems to be invalidated before the first use. Why? -// if (!valid.get()) { -// throw new ParseException(); -// } - return parseResult.rootNode(); - } - - @Override - protected void invalidate() { - valid.set(false); - } - - @Override - public List getDiagnostics() { - return diagnostics; - } - - public static NetbeansRustParserResult complete(Snapshot snapshot, RustParser.Result parseResult) { - return new NetbeansRustParserResult(snapshot, parseResult, parseResult.syntaxErrors().stream().map(ex -> toError(snapshot, ex)).collect(toList())); - } - - private static DefaultError toError(Snapshot snapshot, com.github.drrb.rust.netbeans.parsing.javacc.ParseException e) { - RustToken currentToken = (RustToken) e.currentToken.next; - FileObject file = snapshot.getSource().getFileObject(); - return new DefaultError( - "rust.parse.message", - "Parse error", - e.getMessage(), - file, - currentToken.absoluteBeginPosition - 1, - currentToken.absoluteEndPosition - 1, - false, - Severity.ERROR - ); - } - - public boolean isFailure() throws ParseException { - return rootNode() != null; - } - } -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/OffsetRustToken.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/OffsetRustToken.java index b147b47..62aff06 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/OffsetRustToken.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/OffsetRustToken.java @@ -18,6 +18,7 @@ import org.netbeans.api.lexer.Token; import org.netbeans.api.lexer.TokenHierarchy; +import org.netbeans.api.lexer.TokenId; import org.netbeans.api.lexer.TokenSequence; import org.netbeans.modules.csl.api.OffsetRange; @@ -26,16 +27,16 @@ */ public class OffsetRustToken { - public static OffsetRustToken atCurrentLocation(TokenSequence tokenSequence) { + public static OffsetRustToken atCurrentLocation(TokenSequence tokenSequence) { return new OffsetRustToken(tokenSequence.offsetToken()); } - private final Token token; + private final Token token; - private OffsetRustToken(Token offsetToken) { + private OffsetRustToken(Token offsetToken) { this.token = offsetToken; } - public RustTokenId id() { + public TokenId id() { return token.id(); } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLanguageHierarchy.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLanguageHierarchy.java deleted file mode 100644 index 61b326d..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLanguageHierarchy.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing; - -import com.github.drrb.rust.netbeans.RustLanguage; -import org.netbeans.spi.lexer.LanguageHierarchy; -import org.netbeans.spi.lexer.Lexer; -import org.netbeans.spi.lexer.LexerRestartInfo; - -import java.util.Collection; -import java.util.EnumSet; - -import static java.util.Collections.unmodifiableSet; - -public class RustLanguageHierarchy extends LanguageHierarchy { - private static final Collection TOKEN_IDS = unmodifiableSet(EnumSet.allOf(RustTokenId.class)); - - @Override - protected Collection createTokenIds() { - return TOKEN_IDS; - } - - @Override - protected Lexer createLexer(LexerRestartInfo info) { - return new NetbeansRustLexer(info); - } - - @Override - protected String mimeType() { - return RustLanguage.MIME_TYPE; - } -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustTokenId.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/RustTokenId.java deleted file mode 100644 index 973ca70..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustTokenId.java +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing; - -import com.github.drrb.rust.netbeans.parsing.javacc.RustParserConstants; -import org.netbeans.api.lexer.Language; -import org.netbeans.api.lexer.TokenId; - -public enum RustTokenId implements TokenId { - - EOF(RustParserConstants.EOF, TokenCategory.WHITESPACE), - WHITESPACE(RustParserConstants.WHITESPACE, TokenCategory.WHITESPACE), - DOC_COMMENT(RustParserConstants.DOC_COMMENT, TokenCategory.COMMENT), - INNER_DOC_COMMENT(RustParserConstants.INNER_DOC_COMMENT, TokenCategory.COMMENT), - LINE_COMMENT(RustParserConstants.LINE_COMMENT, TokenCategory.COMMENT), - BLOCK_COMMENT(RustParserConstants.BLOCK_COMMENT, TokenCategory.COMMENT), - DOC_BLOCK_COMMENT(RustParserConstants.DOC_BLOCK_COMMENT, TokenCategory.COMMENT), - INNER_DOC_BLOCK_COMMENT(RustParserConstants.INNER_DOC_BLOCK_COMMENT, TokenCategory.COMMENT), - NON_NULL(RustParserConstants.NON_NULL, TokenCategory.IDENTIFIER), - NON_SINGLE_QUOTE(RustParserConstants.NON_SINGLE_QUOTE, TokenCategory.IDENTIFIER), - NON_DOUBLE_QUOTE(RustParserConstants.NON_DOUBLE_QUOTE, TokenCategory.IDENTIFIER), - NON_EOL(RustParserConstants.NON_EOL, TokenCategory.IDENTIFIER), - ASCII(RustParserConstants.ASCII, TokenCategory.IDENTIFIER), - ASCII_NON_EOL(RustParserConstants.ASCII_NON_EOL, TokenCategory.IDENTIFIER), - ASCII_NON_SINGLE_QUOTE(RustParserConstants.ASCII_NON_SINGLE_QUOTE, TokenCategory.IDENTIFIER), - ASCII_NON_DOUBLE_QUOTE(RustParserConstants.ASCII_NON_DOUBLE_QUOTE, TokenCategory.IDENTIFIER), - STRING_LITERAL(RustParserConstants.STRING_LITERAL, TokenCategory.STRING), - RAW_STRING_LITERAL(RustParserConstants.RAW_STRING_LITERAL, TokenCategory.STRING), - CHAR_LITERAL(RustParserConstants.CHAR_LITERAL, TokenCategory.CHARACTER), - NUMBER_LITERAL(RustParserConstants.NUMBER_LITERAL, TokenCategory.NUMBER), - BYTE_LITERAL(RustParserConstants.BYTE_LITERAL, TokenCategory.CHARACTER), - BYTE_STRING_LITERAL(RustParserConstants.BYTE_STRING_LITERAL, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL(RustParserConstants.RAW_BYTE_STRING_LITERAL, TokenCategory.STRING), - STRING_BODY(RustParserConstants.STRING_BODY, TokenCategory.STRING), - CHAR_BODY(RustParserConstants.CHAR_BODY, TokenCategory.CHARACTER), - BYTE_BODY(RustParserConstants.BYTE_BODY, TokenCategory.IDENTIFIER), - COMMON_ESCAPE(RustParserConstants.COMMON_ESCAPE, TokenCategory.IDENTIFIER), - UNICODE_ESCAPE(RustParserConstants.UNICODE_ESCAPE, TokenCategory.IDENTIFIER), - FLOAT_SUFFIX(RustParserConstants.FLOAT_SUFFIX, TokenCategory.NUMBER), - EXPONENT(RustParserConstants.EXPONENT, TokenCategory.NUMBER), - DEC_LIT(RustParserConstants.DEC_LIT, TokenCategory.NUMBER), - HEX_DIGIT(RustParserConstants.HEX_DIGIT, TokenCategory.NUMBER), - OCT_DIGIT(RustParserConstants.OCT_DIGIT, TokenCategory.NUMBER), - DEC_DIGIT(RustParserConstants.DEC_DIGIT, TokenCategory.NUMBER), - NONZERO_DEC(RustParserConstants.NONZERO_DEC, TokenCategory.NUMBER), - RAW_STRING_LITERAL_3(RustParserConstants.RAW_STRING_LITERAL_3, TokenCategory.STRING), - RAW_STRING_LITERAL_2(RustParserConstants.RAW_STRING_LITERAL_2, TokenCategory.STRING), - RAW_STRING_LITERAL_1(RustParserConstants.RAW_STRING_LITERAL_1, TokenCategory.STRING), - RAW_STRING_LITERAL_0(RustParserConstants.RAW_STRING_LITERAL_0, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL_3(RustParserConstants.RAW_BYTE_STRING_LITERAL_3, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL_2(RustParserConstants.RAW_BYTE_STRING_LITERAL_2, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL_1(RustParserConstants.RAW_BYTE_STRING_LITERAL_1, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL_0(RustParserConstants.RAW_BYTE_STRING_LITERAL_0, TokenCategory.STRING), - DOUBLE_COLON(RustParserConstants.DOUBLE_COLON, TokenCategory.SEPARATOR), - ARROW(RustParserConstants.ARROW, TokenCategory.SEPARATOR), - DOUBLE_ARROW(RustParserConstants.DOUBLE_ARROW, TokenCategory.OPERATOR), - HASH(RustParserConstants.HASH, TokenCategory.SEPARATOR), - LEFT_BRACKET(RustParserConstants.LEFT_BRACKET, TokenCategory.SEPARATOR), - RIGHT_BRACKET(RustParserConstants.RIGHT_BRACKET, TokenCategory.SEPARATOR), - LEFT_PAREN(RustParserConstants.LEFT_PAREN, TokenCategory.SEPARATOR), - RIGHT_PAREN(RustParserConstants.RIGHT_PAREN, TokenCategory.SEPARATOR), - LEFT_BRACE(RustParserConstants.LEFT_BRACE, TokenCategory.SEPARATOR), - RIGHT_BRACE(RustParserConstants.RIGHT_BRACE, TokenCategory.SEPARATOR), - COMMA(RustParserConstants.COMMA, TokenCategory.SEPARATOR), - COLON(RustParserConstants.COLON, TokenCategory.SEPARATOR), - PLUS(RustParserConstants.PLUS, TokenCategory.OPERATOR), - DASH(RustParserConstants.DASH, TokenCategory.OPERATOR), - STAR(RustParserConstants.STAR, TokenCategory.OPERATOR), - FORWARD_SLASH(RustParserConstants.FORWARD_SLASH, TokenCategory.OPERATOR), - PERCENT(RustParserConstants.PERCENT, TokenCategory.OPERATOR), - AMPERSAND(RustParserConstants.AMPERSAND, TokenCategory.OPERATOR), - PIPE(RustParserConstants.PIPE, TokenCategory.OPERATOR), - HAT(RustParserConstants.HAT, TokenCategory.OPERATOR), - DOUBLE_AMPERSAND(RustParserConstants.DOUBLE_AMPERSAND, TokenCategory.OPERATOR), - DOUBLE_PIPE(RustParserConstants.DOUBLE_PIPE, TokenCategory.OPERATOR), - LEFT_ANGLE_BRACKET(RustParserConstants.LEFT_ANGLE_BRACKET, TokenCategory.SEPARATOR), - RIGHT_ANGLE_BRACKET(RustParserConstants.RIGHT_ANGLE_BRACKET, TokenCategory.SEPARATOR), - SHIFT_LEFT(RustParserConstants.SHIFT_LEFT, TokenCategory.OPERATOR), - SHIFT_RIGHT(RustParserConstants.SHIFT_RIGHT, TokenCategory.OPERATOR), - LESS_THAN_EQUAL(RustParserConstants.LESS_THAN_EQUAL, TokenCategory.OPERATOR), - GREATER_THAN_EQUAL(RustParserConstants.GREATER_THAN_EQUAL, TokenCategory.OPERATOR), - SEMICOLON(RustParserConstants.SEMICOLON, TokenCategory.SEPARATOR), - DOUBLE_EQUALS(RustParserConstants.DOUBLE_EQUALS, TokenCategory.OPERATOR), - NOT_EQUAL(RustParserConstants.NOT_EQUAL, TokenCategory.OPERATOR), - PLUS_EQUALS(RustParserConstants.PLUS_EQUALS, TokenCategory.OPERATOR), - MINUS_EQUALS(RustParserConstants.MINUS_EQUALS, TokenCategory.OPERATOR), - TIMES_EQUALS(RustParserConstants.TIMES_EQUALS, TokenCategory.OPERATOR), - DIVIDE_EQUALS(RustParserConstants.DIVIDE_EQUALS, TokenCategory.OPERATOR), - MOD_EQUALS(RustParserConstants.MOD_EQUALS, TokenCategory.OPERATOR), - AND_EQUALS(RustParserConstants.AND_EQUALS, TokenCategory.OPERATOR), - OR_EQUALS(RustParserConstants.OR_EQUALS, TokenCategory.OPERATOR), - XOR_EQUALS(RustParserConstants.XOR_EQUALS, TokenCategory.OPERATOR), - SHIFT_LEFT_EQUALS(RustParserConstants.SHIFT_LEFT_EQUALS, TokenCategory.OPERATOR), - SHIFT_RIGHT_EQUALS(RustParserConstants.SHIFT_RIGHT_EQUALS, TokenCategory.OPERATOR), - BANG(RustParserConstants.BANG, TokenCategory.OPERATOR), - EQUALS(RustParserConstants.EQUALS, TokenCategory.OPERATOR), - DOT(RustParserConstants.DOT, TokenCategory.SEPARATOR), - DOUBLE_DOT(RustParserConstants.DOUBLE_DOT, TokenCategory.IDENTIFIER), - DOLLAR(RustParserConstants.DOLLAR, TokenCategory.SEPARATOR), - HASH_ROCKET(RustParserConstants.HASH_ROCKET, TokenCategory.SEPARATOR), - ABSTRACT(RustParserConstants.ABSTRACT, TokenCategory.IDENTIFIER), - ALIGNOF(RustParserConstants.ALIGNOF, TokenCategory.IDENTIFIER), - AS(RustParserConstants.AS, TokenCategory.KEYWORD), - BECOME(RustParserConstants.BECOME, TokenCategory.IDENTIFIER), - BOX(RustParserConstants.BOX, TokenCategory.IDENTIFIER), - BREAK(RustParserConstants.BREAK, TokenCategory.KEYWORD), - CONST(RustParserConstants.CONST, TokenCategory.KEYWORD), - CONTINUE(RustParserConstants.CONTINUE, TokenCategory.KEYWORD), - CRATE(RustParserConstants.CRATE, TokenCategory.KEYWORD), - DO(RustParserConstants.DO, TokenCategory.KEYWORD), - ELSE(RustParserConstants.ELSE, TokenCategory.KEYWORD), - ENUM(RustParserConstants.ENUM, TokenCategory.KEYWORD), - EXTERN(RustParserConstants.EXTERN, TokenCategory.KEYWORD), - FALSE(RustParserConstants.FALSE, TokenCategory.KEYWORD), - FINAL(RustParserConstants.FINAL, TokenCategory.KEYWORD), - FN(RustParserConstants.FN, TokenCategory.KEYWORD), - FOR(RustParserConstants.FOR, TokenCategory.KEYWORD), - IF(RustParserConstants.IF, TokenCategory.KEYWORD), - IMPL(RustParserConstants.IMPL, TokenCategory.KEYWORD), - IN(RustParserConstants.IN, TokenCategory.KEYWORD), - LET(RustParserConstants.LET, TokenCategory.KEYWORD), - LOOP(RustParserConstants.LOOP, TokenCategory.KEYWORD), - MACRO(RustParserConstants.MACRO, TokenCategory.KEYWORD), - MACRO_RULES(RustParserConstants.MACRO_RULES, TokenCategory.IDENTIFIER), - MATCH(RustParserConstants.MATCH, TokenCategory.KEYWORD), - MOD(RustParserConstants.MOD, TokenCategory.IDENTIFIER), - MOVE(RustParserConstants.MOVE, TokenCategory.IDENTIFIER), - MUT(RustParserConstants.MUT, TokenCategory.KEYWORD), - OFFSETOF(RustParserConstants.OFFSETOF, TokenCategory.IDENTIFIER), - OVERRIDE(RustParserConstants.OVERRIDE, TokenCategory.IDENTIFIER), - PRIV(RustParserConstants.PRIV, TokenCategory.KEYWORD), - PROC(RustParserConstants.PROC, TokenCategory.IDENTIFIER), - PUB(RustParserConstants.PUB, TokenCategory.KEYWORD), - PURE(RustParserConstants.PURE, TokenCategory.IDENTIFIER), - REF(RustParserConstants.REF, TokenCategory.IDENTIFIER), - RETURN(RustParserConstants.RETURN, TokenCategory.KEYWORD), - BIG_SELF(RustParserConstants.BIG_SELF, TokenCategory.IDENTIFIER), - SELF(RustParserConstants.SELF, TokenCategory.KEYWORD), - SIZEOF(RustParserConstants.SIZEOF, TokenCategory.KEYWORD), - STATIC(RustParserConstants.STATIC, TokenCategory.KEYWORD), - STRUCT(RustParserConstants.STRUCT, TokenCategory.KEYWORD), - SUPER(RustParserConstants.SUPER, TokenCategory.IDENTIFIER), - TRAIT(RustParserConstants.TRAIT, TokenCategory.KEYWORD), - TRUE(RustParserConstants.TRUE, TokenCategory.KEYWORD), - TYPE(RustParserConstants.TYPE, TokenCategory.KEYWORD), - TYPEOF(RustParserConstants.TYPEOF, TokenCategory.KEYWORD), - UNSAFE(RustParserConstants.UNSAFE, TokenCategory.KEYWORD), - UNSIZED(RustParserConstants.UNSIZED, TokenCategory.KEYWORD), - USE(RustParserConstants.USE, TokenCategory.KEYWORD), - VIRTUAL(RustParserConstants.VIRTUAL, TokenCategory.KEYWORD), - WHERE(RustParserConstants.WHERE, TokenCategory.KEYWORD), - WHILE(RustParserConstants.WHILE, TokenCategory.KEYWORD), - YIELD(RustParserConstants.YIELD, TokenCategory.KEYWORD), - IDENTIFIER(RustParserConstants.IDENTIFIER, TokenCategory.IDENTIFIER), - LABEL(RustParserConstants.LABEL, TokenCategory.IDENTIFIER), - XID_start(RustParserConstants.XID_start, TokenCategory.IDENTIFIER), - XID_continue(RustParserConstants.XID_continue, TokenCategory.IDENTIFIER), - GARBAGE(RustParserConstants.GARBAGE, TokenCategory.IDENTIFIER); - - public static final RustLanguageHierarchy LANGUAGE_HIERARCHY = new RustLanguageHierarchy(); - private static final RustTokenId[] LOOKUP; - static { - int highestValue = 0; - for (RustTokenId kind : values()) { - highestValue = highestValue > kind.javaccKind ? highestValue : kind.javaccKind; - } - LOOKUP = new RustTokenId[highestValue + 1]; - for (RustTokenId kind : values()) { - LOOKUP[kind.javaccKind] = kind; - } - } - - public static Language language() { - return LANGUAGE_HIERARCHY.language(); - } - - private final int javaccKind; - private final TokenCategory category; - - RustTokenId(int javaccKind, TokenCategory category) { - this.javaccKind = javaccKind; - this.category = category; - } - - public static RustTokenId get(int javaccKind) { - RustTokenId kind = LOOKUP[javaccKind]; - if (kind == null) { - throw new IllegalArgumentException("No TokenKind for constant: " + javaccKind); - } - return kind; - } - - @Override - public String primaryCategory() { - return category.getName(); - } -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLanguageHierarchy.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLanguageHierarchy.java new file mode 100644 index 0000000..06e20eb --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLanguageHierarchy.java @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2017 drrb + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustParser; +import com.github.drrb.rust.netbeans.RustLanguage; +import org.netbeans.spi.lexer.LanguageHierarchy; +import org.netbeans.spi.lexer.Lexer; +import org.netbeans.spi.lexer.LexerRestartInfo; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +public class AntlrRustLanguageHierarchy extends LanguageHierarchy { + + public static final AntlrRustLanguageHierarchy INSTANCE = new AntlrRustLanguageHierarchy(); + + final AntlrTokenIDs tokenIds; + public AntlrRustLanguageHierarchy() { + tokenIds = AntlrTokenIDs.forVocabulary(RustParser.VOCABULARY, RustAntlrLexer::categoryFor); + } + + @Override + protected Collection createTokenIds() { + return tokenIds.all(); + } + + @Override + protected Lexer createLexer(LexerRestartInfo info) { + return new RustAntlrLexer(info); + } + + @Override + protected String mimeType() { + return RustLanguage.MIME_TYPE; + } + + @Override + protected Map> createTokenCategories() { + Map> result = new HashMap<>(); + for (AntlrTokenID id : CommonRustTokenIDs.all()) { + Collection tokens = result.get(id.primaryCategory()); + if (tokens == null) { + tokens = new HashSet<>(); + result.put(id.primaryCategory(), tokens); + } + tokens.add(id); + } + return result; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLexUtils.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLexUtils.java similarity index 69% rename from src/main/java/com/github/drrb/rust/netbeans/parsing/RustLexUtils.java rename to src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLexUtils.java index b8bab8b..40ec117 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLexUtils.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLexUtils.java @@ -14,57 +14,55 @@ * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ -package com.github.drrb.rust.netbeans.parsing; +package com.github.drrb.rust.netbeans.parsing.antlr; +import com.github.drrb.rust.netbeans.parsing.*; import com.github.drrb.rust.netbeans.util.Option; import org.netbeans.api.lexer.TokenHierarchy; import org.netbeans.api.lexer.TokenId; import org.netbeans.api.lexer.TokenSequence; import org.netbeans.modules.csl.api.OffsetRange; import org.netbeans.modules.csl.spi.ParserResult; -import org.netbeans.modules.java.source.usages.DocumentUtil; - -import javax.swing.text.AbstractDocument; import javax.swing.text.BadLocationException; import javax.swing.text.Document; import java.util.logging.Level; import java.util.logging.Logger; -public class RustLexUtils { - private static final Logger LOG = Logger.getLogger(RustLexUtils.class.getName()); +public class AntlrRustLexUtils { + private static final Logger LOG = Logger.getLogger(AntlrRustLexUtils.class.getName()); - public TokenSequence getRustTokenSequence(Document doc, int offset) { + public TokenSequence getRustTokenSequence(Document doc, int offset) { TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc); - TokenSequence topLevelTokenSequence = tokenHierarchy.tokenSequence(RustTokenId.language()); + TokenSequence topLevelTokenSequence = tokenHierarchy.tokenSequence(AntlrRustLanguageHierarchy.INSTANCE.language()); if (topLevelTokenSequence != null) { return topLevelTokenSequence; } - TokenSequence embeddedRustTokenSequence = getEmbeddedRustTokenSequence(tokenHierarchy, offset, true); + TokenSequence embeddedRustTokenSequence = getEmbeddedRustTokenSequence(tokenHierarchy, offset, true); if (embeddedRustTokenSequence != null) { return embeddedRustTokenSequence; } - TokenSequence embeddedRustTokenSequenceForwards = getEmbeddedRustTokenSequence(tokenHierarchy, offset, false); + TokenSequence embeddedRustTokenSequenceForwards = getEmbeddedRustTokenSequence(tokenHierarchy, offset, false); if (embeddedRustTokenSequenceForwards != null) { return embeddedRustTokenSequenceForwards; } try { LOG.warning("Couldn't get Rust token sequence for document. Falling back to lexing it ourselves."); - tokenHierarchy = TokenHierarchy.create(doc.getText(0, doc.getLength()), RustTokenId.language()); - return tokenHierarchy.tokenSequence(RustTokenId.language()); + tokenHierarchy = TokenHierarchy.create(doc.getText(0, doc.getLength()), AntlrRustLanguageHierarchy.INSTANCE.language()); + return tokenHierarchy.tokenSequence(AntlrRustLanguageHierarchy.INSTANCE.language()); } catch (BadLocationException ex) { LOG.log(Level.WARNING, "Couldn't get Rust token sequence for document at all!", ex); return null; } } - private TokenSequence getEmbeddedRustTokenSequence(TokenHierarchy tokenHierarchy, int offset, boolean backwardBias) { + private TokenSequence getEmbeddedRustTokenSequence(TokenHierarchy tokenHierarchy, int offset, boolean backwardBias) { for (TokenSequence tokenSequence : tokenHierarchy.embeddedTokenSequences(offset, backwardBias)) { - if (tokenSequence.language() == RustTokenId.language()) { + if (tokenSequence.language() == AntlrRustLanguageHierarchy.INSTANCE.language()) { @SuppressWarnings("unchecked") - TokenSequence embeddedTokenSequence = (TokenSequence) tokenSequence; + TokenSequence embeddedTokenSequence = (TokenSequence) tokenSequence; return embeddedTokenSequence; } } @@ -77,17 +75,17 @@ public static Option getIdentifierAt(int caretOffset, ParserRes } public static Option getIdentifierAt(int caretOffset, TokenHierarchy tokenHierarchy) { - TokenSequence tokenSequence = tokenHierarchy.tokenSequence(RustTokenId.language()); + TokenSequence tokenSequence = tokenHierarchy.tokenSequence(AntlrRustLanguageHierarchy.INSTANCE.language()); Option tokenAtOffset = offsetTokenAt(caretOffset, tokenSequence); if (tokenAtOffset.isNot()) { return Option.none(); } - - if (tokenAtOffset.value().id() == RustTokenId.IDENTIFIER) { + TokenId identId = CommonRustTokenIDs.identifierTokenID(); + if (tokenAtOffset.value().id() == identId) { return tokenAtOffset; } else if (caretOffset > 0) { Option tokenBeforeOffset = offsetTokenAt(caretOffset - 1, tokenSequence); - if (tokenBeforeOffset.is() && tokenBeforeOffset.value().id() == RustTokenId.IDENTIFIER) { + if (tokenBeforeOffset.is() && tokenBeforeOffset.value().id() == identId) { return tokenBeforeOffset; } else { return Option.none(); @@ -97,7 +95,7 @@ public static Option getIdentifierAt(int caretOffset, TokenHier } } - private static Option offsetTokenAt(int caretPosition, TokenSequence tokenSequence) { + private static Option offsetTokenAt(int caretPosition, TokenSequence tokenSequence) { tokenSequence.move(caretPosition); if (tokenSequence.moveNext()) { return Option.is(OffsetRustToken.atCurrentLocation(tokenSequence)); diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrStreamAdapter.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrStreamAdapter.java new file mode 100644 index 0000000..9380aa7 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrStreamAdapter.java @@ -0,0 +1,162 @@ +/** + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.misc.Interval; +import org.netbeans.spi.lexer.LexerInput; + +public class AntlrStreamAdapter implements CharStream { + + private final String name; + private int index = 0; + private int mark = 0; + private final LexerInput input; + + public AntlrStreamAdapter(LexerInput input, String name) { + this.input = input; + this.name = name; + } + + @Override + public int index() { + return index; + } + + @Override + public int size() { + return -1; + } + + @Override + public String getSourceName() { + return name; + } + + @Override + public void consume() { + int character = read(); + if ( character == EOF ) { + backup( 1 ); + throw new IllegalStateException( "Attempting to consume EOF" ); + } + } + + @Override + public int LA(int lookaheadAmount) { + if ( lookaheadAmount < 0 ) { + return lookBack( -lookaheadAmount ); + } else if ( lookaheadAmount > 0 ) { + return lookAhead( lookaheadAmount ); + } else { + return 0; + } + } + + private int lookBack(int amount) { + backup( amount ); + int character = read(); + for ( int i = 1; i < amount; i++ ) { + read(); + } + return character; + } + + private int lookAhead(int amount) { + int character = 0; + for ( int i = 0; i < amount; i++ ) { + character = read(); + } + backup( amount ); + return character; + } + + @Override + public int mark() { + return ++mark; + } + + @Override + public void release(int marker) { + mark = marker; + mark--; + } + + @Override + public void seek(int index) { + if ( index < 0 ) { + throw new IllegalArgumentException( String.format( "Invalid index (%s < 0)", index ) ); + } + + if ( index < this.index ) { + backup( this.index - index ); + return; + } + while ( this.index < index ) { + consume(); + } + } + + private int read() { + int result = input.read(); + index++; + + if ( result == LexerInput.EOF ) { + return EOF; + } else { + return result; + } + } + + private void backup(int count) { + input.backup( count ); + index -= count; + } + + @Override + public String getText(Interval interval) { + int start = interval.a; + int stop = interval.b; + + if ( start < 0 || stop < start ) { + return ""; + } + + final int pos = this.index; + final int length = interval.length(); + final char[] data = new char[length]; + + seek( interval.a ); + int r = 0; + while ( r < length ) { + final int character = read(); + if ( character == EOF ) { + break; + } + + data[r] = (char) character; + r++; + } + seek( pos ); + + if ( r > 0 ) { + return new String( data, 0, r ); + } else { + return ""; + } + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenID.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenID.java new file mode 100644 index 0000000..e57cf58 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenID.java @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import java.util.Objects; +import org.netbeans.api.lexer.TokenId; + +/** + * Generic TokenId implementation for tokens from an Antlr 4 Vocabulary. + * + * @author Tim Boudreau + */ +public class AntlrTokenID implements TokenId { + + private final int tokenType; + private final String literalName; + private final String displayName; + private final String symbolicName; + private final String category; + + public static AntlrTokenID EOF = new AntlrTokenID(-1, "EOF", "EOF", "EOF", "other"); + + AntlrTokenID(int tokenType, String literalName, String displayName, + String symbolicName, String category) { + this.tokenType = tokenType; + this.literalName = literalName; + this.displayName = displayName; + this.symbolicName = symbolicName; + this.category = category; + } + + public String literalName() { + return literalName; + } + + public String symbolicName() { + return symbolicName(); + } + + public String displayName() { + return displayName; + } + + @Override + public String toString() { + return tokenType + "/" + (symbolicName == null ? "-" : "'" + symbolicName + "'") + + "/" + literalName + "/" + displayName; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o instanceof AntlrTokenID) { + AntlrTokenID other = (AntlrTokenID) o; + return other.tokenType == tokenType && + Objects.equals(other.literalName, literalName); + } + return false; + } + + @Override + public int hashCode() { + return ((tokenType + 1) * 51) * Objects.hashCode(literalName); + } + + @Override + public String name() { + if (symbolicName != null) { + return symbolicName; + } else if (literalName != null) { + return literalName; + } else if (displayName != null) { + return displayName; + } else { + return "????"; + } + } + + @Override + public int ordinal() { + return tokenType; + } + + @Override + public String primaryCategory() { + return category; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenIDs.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenIDs.java new file mode 100644 index 0000000..4397509 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenIDs.java @@ -0,0 +1,239 @@ +/** + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustLexer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.regex.Pattern; +import org.antlr.v4.runtime.Vocabulary; + +/** + * Generic support for generating a set of token IDs from any Antlr 4 + * Vocabulary - not specific to Rust. + * + * @author Tim Boudreau + */ +public class AntlrTokenIDs implements Iterable { + + private static Map mapping + = new WeakHashMap<>(); + + static synchronized AntlrTokenIDs forVocabulary(Vocabulary vocabulary, TokenCategorizer cat) { + AntlrTokenIDs result = mapping.get(vocabulary); + if (result == null) { + result = new AntlrTokenIDs(vocabulary, cat); + mapping.put(vocabulary, result); + } + return result; + } + + private Map byName = new HashMap<>(); + private final AntlrTokenID[] ids; + private final Map byCharacter = new HashMap<>(); + private final Map bySymbolicName = new HashMap<>(); + + AntlrTokenIDs(Vocabulary vocabulary, TokenCategorizer categorizer) { + int end = vocabulary.getMaxTokenType() + 1; + ids = new AntlrTokenID[end]; + for (int i = 0; i < end; i++) { + String displayName = stripSingleQuotes(vocabulary.getDisplayName(i)); + String symName = stripSingleQuotes(vocabulary.getSymbolicName(i)); + String litName = stripSingleQuotes(vocabulary.getLiteralName(i)); + String category = categorizer.categoryFor(i, displayName, + symName, litName); + ids[i] = new AntlrTokenID(i, litName, displayName, symName, category); + if (litName != null) { + byName.put(litName, ids[i]); + if (litName.length() == 1) { + byCharacter.put(litName.charAt(0), ids[i]); + } + } + if (symName != null) { + bySymbolicName.put(symName, ids[i]); + } + } + } + + public AntlrTokenID forSymbolicName(String name) { + if ("EOF".equals(name)) { + return AntlrTokenID.EOF; + } + return bySymbolicName.get(name); + } + + public AntlrTokenID forSymbol(char symbol) { + if (-1 == symbol) { + return AntlrTokenID.EOF; + } + return byCharacter.get(symbol); + } + + static String stripSingleQuotes(String s) { + if (s != null && s.length() > 1) { + char a = s.charAt(0); + char b = s.charAt(s.length() -1); + if (a == '\'' && b == '\'') { + s = s.substring(1, s.length()-1); + } + } + return s; + } + + public AntlrTokenID get(int tokenType) { + if (-1 == tokenType) { + return AntlrTokenID.EOF; + } + assert tokenType >= 0 && tokenType < ids.length : "Invalid token type " + tokenType; + return ids[tokenType]; + } + + public AntlrTokenID get(String literalName) { + if ("EOF".equals(literalName)) { + return AntlrTokenID.EOF; + } + AntlrTokenID result = byName.get(literalName); + if (result == null) { + throw new IllegalArgumentException("No token with symbolic name " + literalName); + } + return result; + } + + public int size() { + return ids.length; + } + + public List all() { + return Arrays.asList(ids); + } + + public Iterator iterator() { + return new ArrayIterator<>(ids); + } + + private static final class ArrayIterator implements Iterator { + + private final T[] arr; + private int index = -1; + + public ArrayIterator(T[] arr) { + this.arr = arr; + } + + @Override + public boolean hasNext() { + return index < arr.length - 1; + } + + @Override + public T next() { + return arr[++index]; + } + } + + public static void main(String[] args) { + TokenCategorizer cat = new TokenCategorizer() { + private final Pattern WORD = Pattern.compile("^[a-zA-Z]+$"); + @Override + public String categoryFor(int tokenType, String displayName, String symbolicName, String literalName) { + if (tokenType == 0) { + return "eof"; + } + System.out.println("DN " + displayName + " SN " + symbolicName + " LN " + literalName + " " + tokenType); + if (literalName != null && WORD.matcher(literalName).lookingAt()) { + return "keyword"; + } else if (literalName != null && literalName.length() == 1 && !Character.isAlphabetic(literalName.charAt(0))) { + switch(literalName.charAt(0)) { + case '*': + case '/': + case '%': + case '+': + case '-': + case '^': + case '|': + case '&': + return "operator"; + case '.' : + case '{': + case '}': + case '(': + case ')': + case '[': + case ']': + case ',': + return "delimiter"; + case '<' : + case '>' : + return "comparisonOperator"; + case '=' : + return "assignmentOperator"; + } + return "symbol"; + } else if (literalName != null && literalName.length() == 2 && !Character.isAlphabetic(literalName.charAt(0)) && !Character.isAlphabetic(literalName.charAt(1))) { + switch(literalName) { + case "::" : + case "=>": + return "delimiter"; + case "+=" : + case "-=" : + case "/=" : + case "*=" : + case "%=" : + case "|=" : + case "&=" : + case "^=" : + return "assignmentOperator"; + case "==": + return "comparisonOperator"; + + } + return "symbol"; + } else if (literalName != null && literalName.length() == 3 && !Character.isAlphabetic(literalName.charAt(0)) && !Character.isAlphabetic(literalName.charAt(1)) && !Character.isAlphabetic(literalName.charAt(2))) { + switch(literalName) { + case "<<=" : + case ">>=" : + return "assignmentOperator"; + } + return "symbol"; + } else if (symbolicName != null) { + if (symbolicName.endsWith("Comment")) { + return "comment"; + } else if (symbolicName.endsWith("Lit")) { + return "literal"; + } + switch(symbolicName) { + case "Lifetime" : + return "keyword"; + case "Whitespace": + return "whitespace"; + case "Ident": + return "identifier"; + } + } + return "other"; + } + }; + AntlrTokenIDs ids = new AntlrTokenIDs(RustLexer.VOCABULARY, cat); + for (AntlrTokenID id : ids) { + System.out.println(id.primaryCategory() + "\t\t" + id.name()); + } + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrUtils.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrUtils.java new file mode 100644 index 0000000..bee6aac --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrUtils.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import java.util.Arrays; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.ParseTree; +import org.netbeans.modules.csl.api.OffsetRange; + +/** + * + * @author Tim Boudreau + */ +public final class AntlrUtils { + + private AntlrUtils() { + throw new AssertionError(); + } + + public static OffsetRange toOffsetRange(ParserRuleContext ctx) { + return new OffsetRange( + ctx.getStart().getStartIndex(), + ctx.getStop().getStopIndex() + 1); + } + + public static void print(ParserRuleContext ctx) { + StringBuilder sb = new StringBuilder("\n*******************************\n") + .append(ctx.getText()).append('\n'); + unwind(ctx, sb); + System.out.println(sb.toString()); + } + + public static String stringify(ParserRuleContext ctx) { + StringBuilder sb = new StringBuilder(); + unwind(ctx, sb); + return sb.toString(); + } + + private static void unwind(ParserRuleContext ctx, StringBuilder sb) { + unwind(ctx, 0, sb); + } + + private static void unwind(ParseTree ctx, int depth, StringBuilder sb) { + char[] ind = new char[depth * 2]; + Arrays.fill(ind, ' '); + sb.append(ind); + sb.append(ctx.getClass().getSimpleName()).append(" - ").append(ctx.getText()); + if (ctx instanceof ParserRuleContext) { + ParserRuleContext rule = (ParserRuleContext) ctx; + if (rule.children == null || rule.children.isEmpty()) { + sb.append(" with no children\n"); + } else { + if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '\n') { + sb.append('\n'); + } + for (ParseTree c : rule.children) { + unwind(c, depth + 1, sb); + if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '\n') { + sb.append('\n'); + } + } + } + } + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/CommonRustTokenIDs.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/CommonRustTokenIDs.java new file mode 100644 index 0000000..81855e8 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/CommonRustTokenIDs.java @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustLexer; +import static com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLanguageHierarchy.INSTANCE; + +/** + * + * @author Tim Boudreau + */ +public class CommonRustTokenIDs { + + public static AntlrTokenID stringLiteral() { + return forTokenType(RustLexer.StringLiteral); + } + + public static AntlrTokenID function() { + return forTokenType(RustLexer.Fn); + } + + public static AntlrTokenID eof() { + return AntlrTokenID.EOF; + } + + public static AntlrTokenID forSymbol(char symbol) { + return INSTANCE.tokenIds.forSymbol(symbol); + } + + public static AntlrTokenID forSymbolicName(String symbolicName) { + return INSTANCE.tokenIds.forSymbolicName(symbolicName); + } + + public static AntlrTokenID forLiteralName(String name) { + return INSTANCE.tokenIds.get(name); + } + + public static AntlrTokenID forTokenType(int id) { + return INSTANCE.tokenIds.get(id); + } + + public static AntlrTokenID whitespaceTokenID() { + return forTokenType(RustLexer.Whitespace); + } + + public static AntlrTokenID identifierTokenID() { + return forSymbolicName("Ident"); + } + + private static AntlrTokenID bang; + public static AntlrTokenID bang() { + return bang == null ? bang = forSymbol('!') : bang; + } + + private static AntlrTokenID semicolon; + public static AntlrTokenID semicolon() { + return semicolon == null ? semicolon = forTokenType(RustLexer.Semicolon) : semicolon; + } + + private static AntlrTokenID leftBrace; + public static AntlrTokenID leftBrace() { + return leftBrace == null ? leftBrace = forTokenType(RustLexer.LeftBrace) : leftBrace; + } + + private static AntlrTokenID rightBrace; + public static AntlrTokenID rightBrace() { + return rightBrace == null ? rightBrace = forTokenType(RustLexer.RightBrace) : rightBrace; + } + + private static AntlrTokenID leftParen; + public static AntlrTokenID leftParen() { + return leftParen == null ? leftParen = forTokenType(RustLexer.LeftParen) : leftParen; + } + + private static AntlrTokenID rightParen; + public static AntlrTokenID rightParen() { + return rightParen == null ? rightParen = forTokenType(RustLexer.RightParen) : rightParen; + } + + private static AntlrTokenID leftBracket; + public static AntlrTokenID leftBracket() { + return leftBracket== null ? leftBracket= forTokenType(RustLexer.LeftBracket) : leftBracket; + } + + private static AntlrTokenID rightBracket; + public static AntlrTokenID rightBracket() { + return rightBracket == null ? rightBracket= forTokenType(RustLexer.RightBracket) : rightBracket; + } + + private static AntlrTokenID leftAngleBracket; + public static AntlrTokenID leftAngleBracket() { + return leftAngleBracket== null ? leftAngleBracket= forTokenType(RustLexer.LeftAngleBracket) : leftAngleBracket; + } + + private static AntlrTokenID rightAngleBracket; + public static AntlrTokenID rightAngleBracket() { + return rightAngleBracket == null ? rightAngleBracket= forTokenType(RustLexer.RightAngleBracket) : rightAngleBracket; + } + + private static AntlrTokenID comma; + public static AntlrTokenID comma() { + return comma == null ? comma= forTokenType(RustLexer.Comma) : comma; + } + + public static Iterable all() { + return INSTANCE.tokenIds; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/ErrImpl.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/ErrImpl.java new file mode 100644 index 0000000..ab2e53a --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/ErrImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.Severity; +import org.netbeans.modules.parsing.api.Snapshot; +import org.openide.filesystems.FileObject; + +/** + * Implementation of Error which wraps an ErrorNode encountered in a + * parse tree, distinguishable from syntax errors. + * + * @author Tim Boudreau + */ +final class ErrImpl implements Error { + + private final Severity severity; + private final String message; + private final FileObject file; + private final int start; + private final int end; + private final boolean line; + + ErrImpl(ErrorNode nd, Snapshot snaphsot) { + this(nd, Severity.FATAL, snaphsot); + } + + ErrImpl(ErrorNode nd, Severity severity, Snapshot snapshot) { + // Ensure we do not hold the snapshot or any objects from + // the parse, as these will be de-facto memory leaks + this.severity = severity; + message = nd.toString(); + file = snapshot.getSource().getFileObject(); + Interval interval = nd.getSourceInterval(); + int start = interval.a; + int end = interval.b; + if (start == -1) { + start = nd.getSymbol().getStartIndex(); + end = nd.getSymbol().getStopIndex() + 1; + } + // Negative starts will go boom, and start must be > end + // to produce highlighting + this.start = Math.max(0, start); + this.end = Math.max(1, end); + line = nd.getText() != null && !nd.getText().trim().contains("\n"); + } + + @Override + public int getStartPosition() { + return start; + } + + @Override + public int getEndPosition() { + return end; + } + + @Override + public boolean isLineError() { + return line; + } + + @Override + public String getDisplayName() { + return getDescription(); + } + + @Override + public String getDescription() { + return message; + } + + @Override + public String getKey() { + return getStartPosition() + ":" + getEndPosition() + ":" + getDescription(); + } + + @Override + public FileObject getFile() { + return file; + } + + @Override + public Severity getSeverity() { + return severity; + } + + @Override + public Object[] getParameters() { + // XXX what is this for? + return new Object[0]; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAnalyzer.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAnalyzer.java new file mode 100644 index 0000000..c6ca8da --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAnalyzer.java @@ -0,0 +1,499 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustBaseVisitor; +import com.github.drrb.rust.antlr.RustParser; +import com.github.drrb.rust.antlr.RustParser.BlockContext; +import static com.github.drrb.rust.netbeans.parsing.antlr.AntlrUtils.toOffsetRange; +import java.util.BitSet; +import java.util.EnumSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import org.antlr.v4.runtime.ANTLRErrorListener; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenSource; +import org.antlr.v4.runtime.atn.ATNConfigSet; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.runtime.misc.Pair; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.netbeans.modules.parsing.api.Snapshot; + +/** + * + * @author Tim Boudreau + */ +class RustAnalyzer extends RustBaseVisitor { + + private final RustParseInfo info; + private final Snapshot snapshot; + private final AtomicBoolean cancelled; + + RustAnalyzer(Snapshot snapshot, AtomicBoolean cancelled) { + this.info = new RustParseInfo(); + this.snapshot = snapshot; + this.cancelled = cancelled; + } + + @Override + public Void visit(ParseTree tree) { + if (cancelled.get()) { + return null; + } + return super.visit(tree); //To change body of generated methods, choose Tools | Templates. + } + + static RustParseInfo analyze(RustParser parser, Snapshot snapshot, AtomicBoolean cancelled) { + RustAnalyzer result = new RustAnalyzer(snapshot, cancelled); + parser.setErrorHandler(new ErrStrategy()); + parser.addErrorListener(new ErrorCapturer(snapshot, parser, result.info)); + parser.crate().accept(result); + return result.info; + } + + static boolean FULL_MESSAGES = Boolean.getBoolean("rust.antlr.full.messages"); + + static class ErrStrategy extends DefaultErrorStrategy { + + // Report full error messages for some tests + @Override + protected Token getMissingSymbol(Parser recognizer) { + Token oldRes = super.getMissingSymbol(recognizer); + if (true) { + return oldRes; + } + Token curr = recognizer.getCurrentToken(); +// System.out.println("oldRes type " + oldRes.getType() + " index " + oldRes.getTokenIndex()); + if (oldRes.getTokenIndex() == -1) { + IntervalSet expecting = getExpectedTokens(recognizer); + int expectedTokenType = Token.INVALID_TYPE; + if (!expecting.isNil()) { +// System.out.println("EXPECTING " + expecting.toString(recognizer.getVocabulary())); + expectedTokenType = expecting.getMinElement(); // get any element + } +// System.out.println("EXPECTED TOKEN TYPE " + recognizer.getVocabulary().getDisplayName(nextTokensState)); + String toInsert = null; + switch (curr.getText()) { + case "(": + toInsert = ")"; + break; + case "[": + toInsert = "]"; + break; + case "{": + toInsert = "}"; + break; + case "<": + toInsert = ">"; + break; + } + if (toInsert != null) { + System.out.println("CONJURE MISSING TOKEN " + toInsert + + " for " + curr.getText() + " index " + + curr.getTokenIndex() + " type " + curr.getType()); + + int start = curr.getStartIndex(); + int stop = curr.getStopIndex(); + return recognizer.getTokenFactory().create(new Pair( + curr.getTokenSource(), curr.getTokenSource().getInputStream()), + expectedTokenType, toInsert, + Token.DEFAULT_CHANNEL, + start, stop, + curr.getLine(), curr.getCharPositionInLine()); + } + } + + System.out.println("GET MISSING SYMBOL at " + recognizer.getCurrentToken() + + " super returns " + oldRes); + return oldRes; + } + + @Override + protected void reportNoViableAlternative(Parser recognizer, NoViableAltException e) { + Token offending = e.getOffendingToken(); + if (offending.getText().length() == 1 && !FULL_MESSAGES) { + recognizer.notifyErrorListeners(offending, "Unexpected symbol: '" + offending.getText() + "'", e); + } else { + super.reportNoViableAlternative(recognizer, e); + } + } + + @Override + protected void reportUnwantedToken(Parser recognizer) { + if (FULL_MESSAGES) { + super.reportUnwantedToken(recognizer); + return; + } + if (inErrorRecoveryMode(recognizer)) { + return; + } + + beginErrorCondition(recognizer); + + Token t = recognizer.getCurrentToken(); + String tokenName = getTokenErrorDisplay(t); + IntervalSet expecting = getExpectedTokens(recognizer); + String msg; + // Keep error messages to a reasonable size - if we're going + // to print out every keyword in the language, that doesn't + // help anyone + if (expecting.size() > 4) { + msg = "extraneous input " + tokenName; + } else { + msg = "extraneous input " + tokenName + " expecting " + + expecting.toString(recognizer.getVocabulary()); + } + recognizer.notifyErrorListeners(t, msg, null); + } + } + + static class ErrorCapturer implements ANTLRErrorListener { + + private final Snapshot snapshot; + private final RustParser parser; + private final RustParseInfo info; + + public ErrorCapturer(Snapshot snapshot, RustParser parser, RustParseInfo info) { + this.snapshot = snapshot; + this.parser = parser; + this.info = info; + } + + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, + int line, + int charPositionInLine, + String msg, + RecognitionException e) { + org.antlr.v4.runtime.Token currentToken = parser.getCurrentToken(); + if (!FULL_MESSAGES) { + switch (msg) { + case "extraneous input ''": + msg = "Premature end of file"; + } + } + info.addError(new SyntaxError(currentToken, msg, snapshot.getSource().getFileObject())); + +// info.addError(new DefaultError(currentToken.toString(), "Syntax error", msg, +// snapshot.getSource().getFileObject(), +// currentToken.getStartIndex(), currentToken.getStopIndex() + 1, +// true, Severity.FATAL)); + } + + @Override + public void reportAmbiguity(Parser parser, DFA dfa, int i, int i1, boolean bln, BitSet bitset, ATNConfigSet atncs) { +// System.out.println("AE ambiguity at " + i + ":" + i1 + " " + dfa.toLexerString() +// + " " + parser.getCurrentToken()); +// String exp = parser.getExpectedTokens().toString(parser.getVocabulary()); +// System.out.println("EXPECTED: " + exp); + } + + @Override + public void reportAttemptingFullContext(Parser parser, DFA dfa, int i, int i1, BitSet bitset, ATNConfigSet atncs) { +// System.out.println("AE attempt full context at " + i + ":" + i1); + } + + @Override + public void reportContextSensitivity(Parser parser, DFA dfa, int i, int i1, int i2, ATNConfigSet atncs) { +// System.out.println("AE context sensitivity at " + i + ":" + i1); + } + + } + + @Override + public Void visitField(RustParser.FieldContext ctx) { + return super.visitField(ctx); + } + + @Override + public Void visitAttr(RustParser.AttrContext ctx) { + info.addSemanticRegion(RustElementKind.ATTR, toOffsetRange(ctx)); + info.addStructureItem(new RustStructureItemImpl(ctx.toString(), RustElementKind.ATTR, snapshot, ctx)); + return super.visitAttr(ctx); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public Void visitField_decl(RustParser.Field_declContext ctx) { + String id = findIdentifier(ctx, RustElementKind.FIELD); + if (id != null) { + info.addStructureItem(new RustStructureItemImpl(id, RustElementKind.FIELD, snapshot, ctx)); + } + return super.visitField_decl(ctx); + } + + @Override + public Void visitEnum_variant(RustParser.Enum_variantContext ctx) { + String id = findIdentifier(ctx, RustElementKind.ENUM_CONSTANT); + if (id != null) { + info.addStructureItem(new RustStructureItemImpl(id, RustElementKind.FIELD, snapshot, ctx)); + } + return super.visitEnum_variant(ctx); //To change body of generated methods, choose Tools | Templates. + } + + public static void unwind(ParserRuleContext ctx) { + AntlrUtils.print(ctx); + } + + @Override + public Void visitFn_head(RustParser.Fn_headContext ctx) { + String id = findIdentifier(ctx, RustElementKind.FUNCTION); + if (id != null) { + id = id.trim(); + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.FUNCTION, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitFn_head(ctx); + }); + } else { + super.visitFn_head(ctx); + } + return null; + } + + @Override + public Void visitLifetime_list(RustParser.Lifetime_listContext ctx) { + info.addSemanticRegion(RustElementKind.LIFETIME, toOffsetRange(ctx)); + return super.visitLifetime_list(ctx); + } + + @Override + public Void visitEnum_decl(RustParser.Enum_declContext ctx) { + String id = findIdentifier(ctx, RustElementKind.ENUM); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.ENUM, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitEnum_decl(ctx); + }); + } else { + super.visitEnum_decl(ctx); + } + return null; + } + + @Override + public Void visitType_decl(RustParser.Type_declContext ctx) { + String id = findIdentifier(ctx, RustElementKind.TYPE); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.TYPE, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitType_decl(ctx); + }); + } else { + super.visitType_decl(ctx); + } + return null; + } + + @Override + public Void visitImpl_block(RustParser.Impl_blockContext ctx) { + String id = findIdentifier(ctx, RustElementKind.IMPL); +// System.out.println("IMPL BLOCK FOUND ID " + id); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.IMPL, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitImpl_block(ctx); + }); + } else { + super.visitImpl_block(ctx); + } + return null; + } + + @Override + public Void visitStruct_decl(RustParser.Struct_declContext ctx) { + String id = findIdentifier(ctx.getParent(), RustElementKind.STRUCT); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.STRUCT, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitStruct_decl(ctx); + }); + } else { + super.visitStruct_decl(ctx); + } + return null; + } + + @Override + public Void visitMacro_tail(RustParser.Macro_tailContext ctx) { + return super.visitMacro_tail(ctx); + } + + @Override + public Void visitTrait_decl(RustParser.Trait_declContext ctx) { + String id = findIdentifier(ctx, RustElementKind.TRAIT); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.TRAIT, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitTrait_decl(ctx); + }); + } else { + super.visitTrait_decl(ctx); + } + return null; + } + + private final IdentifierFinder idFinder = new IdentifierFinder(); + + private String findIdentifier(ParserRuleContext ctx, RustElementKind kind) { + String result = idFinder.find(ctx); + if (result != null) { + info.addSemanticRegion(idFinder.name, kind, toOffsetRange(idFinder.context), + idFinder.mutable, idFinder.visibility(), idFinder.statyc); + } + return result; + } + + static final class IdentifierFinder extends RustBaseVisitor { + + private String name; + private RustParser.IdentContext context; + private boolean mutable; + private boolean statyc; + private final Set visibility = EnumSet + .noneOf(RustVisibility.class); + + Set visibility() { + return EnumSet.copyOf(visibility); + } + + void reset() { + name = null; + context = null; + mutable = false; + statyc = false; + visibility.clear(); + } + + String find(ParserRuleContext ctx) { + reset(); + ctx.accept(this); + return this.name; + } + + @Override + public Void visit(ParseTree tree) { + if (name != null) { + return null; + } + return super.visit(tree); + } + + private void maybeAddVisibility(RustVisibility vis) { + if (vis != null) { + visibility.add(vis); + } + } + + boolean inVisitVisibility; + + @Override + public Void visitVisibility(RustParser.VisibilityContext ctx) { + inVisitVisibility = true; + try { + RustParser.Visibility_restrictionContext vr = ctx.visibility_restriction(); + if (vr != null) { + Token stop = vr.stop; + TokenSource src = vr.start.getTokenSource(); + CharStream in = src.getInputStream(); + for (Token tk = src.nextToken(); in.index() < in.size() && tk.getStopIndex() <= stop.getStopIndex(); tk = src.nextToken()) { + maybeAddVisibility(RustVisibility.forToken(tk)); + switch (tk.getText()) { + // XXX could look these up faster by token type, but + // because they aren't named in the grammar, they are, + // e.g. T__13, which is not a stable identifier. If + // we wind up patching the grammar, this is something + // to fix + case "pub": + case "crate": + case "super": + case "in": + } + } + } + return super.visitVisibility(ctx); + } finally { + inVisitVisibility = false; + } + } + + @Override + public Void visitIdent(RustParser.IdentContext ctx) { + if (inVisitVisibility) { + // "in" visibility will have an identifier which + // is not the one we want + return super.visitIdent(ctx); + } + if (name != null) { + return null; + } + assert name == null && context == null : "Already called with " + name; + name = ctx.getText(); + context = ctx; + // Don't call super - we're done visiting after we find + // the identifier + return null; + } + + @Override + public Void visitStatic_decl(RustParser.Static_declContext ctx) { + statyc = true; + return super.visitStatic_decl(ctx); + } + + @Override + public Void visitMut_or_const(RustParser.Mut_or_constContext ctx) { + if ("mut".equals(ctx.start.getText())) { + mutable = true; + } + return super.visitMut_or_const(ctx); + } + } + + @Override + public Void visitBlock(BlockContext ctx) { + info.addBlock(ctx); + return super.visitBlock(ctx); + } + + @Override + public Void visitTerminal(TerminalNode tn) { + return super.visitTerminal(tn); + } + + @Override + public Void visitErrorNode(ErrorNode en) { + if (en.getSourceInterval().a != -1) { + info.addError(new ErrImpl(en, snapshot)); + } + return super.visitErrorNode(en); + } + + @Override + public Void visitFn_decl(RustParser.Fn_declContext ctx) { +// unwind(ctx); + return super.visitFn_decl(ctx); //To change body of generated methods, choose Tools | Templates. + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrLexer.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrLexer.java new file mode 100644 index 0000000..8a0d4a2 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrLexer.java @@ -0,0 +1,160 @@ +/** + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustLexer; +import java.util.regex.Pattern; +import org.antlr.v4.runtime.CharStreams; +import org.netbeans.api.lexer.PartType; +import org.netbeans.api.lexer.Token; +import org.netbeans.spi.lexer.Lexer; +import org.netbeans.spi.lexer.LexerRestartInfo; + +/** + * + * @author Tim Boudreau + */ +public class RustAntlrLexer implements Lexer { + + public static com.github.drrb.rust.netbeans.rustbridge.RustLexer forString(String input) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + private final AntlrTokenIDs tokenIds; + private LexerRestartInfo info; + private RustLexer lexer; + public RustAntlrLexer() { + tokenIds = AntlrTokenIDs.forVocabulary(RustLexer.VOCABULARY, RustAntlrLexer::categoryFor); + } + + RustAntlrLexer(LexerRestartInfo info) { + tokenIds = AntlrTokenIDs.forVocabulary(RustLexer.VOCABULARY, RustAntlrLexer::categoryFor); + this.info = info; + lexer = new RustLexer( new AntlrStreamAdapter( info.input(), "RustAntlrLexer" ) ); +// lexer = new RustLexer(CharStreams.fromString(info.input().readText().toString())); + } + + public static RustLexer fromString(String s) { + return new RustLexer(CharStreams.fromString(s)); + } + + @Override + public Token nextToken() { + org.antlr.v4.runtime.Token antlrToken = lexer.nextToken(); + AntlrTokenID id; + if ( antlrToken.getType() == RustLexer.EOF && antlrToken.getStopIndex() < antlrToken.getStartIndex() ) { + return null; + } +// if ( info.input().readLength() < 1 ) { +// return null; // XXX eof? +// } + assert antlrToken.getType() <= RustLexer.VOCABULARY.getMaxTokenType(); + id = tokenIds.get( antlrToken.getType() ); + Token tok = info.tokenFactory().createToken( id, ( antlrToken.getStopIndex() + - antlrToken.getStartIndex() ) + 1, PartType.COMPLETE ); + return tok; + } + + @Override + public Object state() { + return null; + } + + @Override + public void release() { + lexer = null; + info = null; + } + + private static final Pattern WORD = Pattern.compile("^[a-zA-Z]+$"); + static String categoryFor(int tokenType, String displayName, String symbolicName, String literalName) { + if (tokenType == 0) { + return "eof"; + } + if (literalName != null && WORD.matcher(literalName).lookingAt()) { + return "keyword"; + } else if (literalName != null && literalName.length() == 1 && !Character.isAlphabetic(literalName.charAt(0))) { + switch (literalName.charAt(0)) { + case '*': + case '/': + case '%': + case '+': + case '-': + case '^': + case '|': + case '&': + return "operator"; + case '.': + case '{': + case '}': + case '(': + case ')': + case '[': + case ']': + case ',': + return "delimiter"; + case '<': + case '>': + return "comparisonOperator"; + case '=': + return "assignmentOperator"; + } + return "symbol"; + } else if (literalName != null && literalName.length() == 2 && !Character.isAlphabetic(literalName.charAt(0)) && !Character.isAlphabetic(literalName.charAt(1))) { + switch (literalName) { + case "::": + case "=>": + return "delimiter"; + case "+=": + case "-=": + case "/=": + case "*=": + case "%=": + case "|=": + case "&=": + case "^=": + return "assignmentOperator"; + case "==": + return "comparisonOperator"; + + } + return "symbol"; + } else if (literalName != null && literalName.length() == 3 && !Character.isAlphabetic(literalName.charAt(0)) && !Character.isAlphabetic(literalName.charAt(1)) && !Character.isAlphabetic(literalName.charAt(2))) { + switch (literalName) { + case "<<=": + case ">>=": + return "assignmentOperator"; + } + return "symbol"; + } else if (symbolicName != null) { + if (symbolicName.endsWith("Comment")) { + return "comment"; + } else if (symbolicName.endsWith("Lit")) { + return "literal"; + } + switch (symbolicName) { + case "Lifetime": + return "keyword"; + case "Whitespace": + return "whitespace"; + case "Ident": + return "identifier"; + } + } + return "other"; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParser.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParser.java new file mode 100644 index 0000000..1119c18 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParser.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustLexer; +import com.github.drrb.rust.antlr.RustParser; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.swing.event.ChangeListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.netbeans.modules.parsing.api.Snapshot; +import org.netbeans.modules.parsing.api.Task; +import org.netbeans.modules.parsing.spi.ParseException; +import org.netbeans.modules.parsing.spi.Parser; +import org.netbeans.modules.parsing.spi.SourceModificationEvent; + +/** + * + * @author Tim Boudreau + */ +public class RustAntlrParser extends Parser { + private RustAntlrParserResult result; + private AtomicBoolean cancelled = new AtomicBoolean(); + + public RustParser parseString(String source) { + RustLexer lexer = new RustLexer(CharStreams.fromString(source)); + return new RustParser(new CommonTokenStream(lexer, 0)); + } + + @Override + public void parse(Snapshot snpsht, Task task, SourceModificationEvent sme) throws ParseException { + cancelled.set(false); + String source = snpsht.getText().toString(); + RustParser parser = parseString(source); + RustAntlrParserResult result = new RustAntlrParserResult(snpsht, parser, cancelled); + synchronized(this) { + this.result = result; + } +// System.out.println("PARSE RESULT " + result); + } + + @Override + public void cancel(CancelReason cr, SourceModificationEvent sme) { + System.out.println("cancelled because of " + cr); + cancel(); + } + + @Override + public void cancel() { + System.out.println("parse cancelled"); + cancelled.set(true); + RustAntlrParserResult result; + synchronized(this) { + result = this.result; + } + if (result != null) { + result.invalidate(); + } + } + + + @Override + public Result getResult(Task task) throws ParseException { + return result; + } + + @Override + public void addChangeListener(ChangeListener cl) { + // do nothing + } + + @Override + public void removeChangeListener(ChangeListener cl) { + // do nothing + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParserResult.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParserResult.java new file mode 100644 index 0000000..229c7b2 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParserResult.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustParser; +import com.github.drrb.rust.antlr.RustVisitor; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.spi.ParserResult; +import org.netbeans.modules.parsing.api.Snapshot; + +/** + * + * @author Tim Boudreau + */ +public final class RustAntlrParserResult extends ParserResult { + + private final RustParseInfo info; + private RustParser parser; + + public RustAntlrParserResult(Snapshot snapshot, RustParser parser, AtomicBoolean cancelled) { + super(snapshot); + this.parser = parser; + info = RustAnalyzer.analyze(parser, snapshot, cancelled); + } + + public String toString() { + return "RustAntlrParserResult {" + info + "}"; + } + + @SuppressWarnings("null") + public boolean accept(RustVisitor visitor) { + RustParser parserLocal; + synchronized(this) { + parserLocal = this.parser; + } + boolean result = parserLocal != null && parserLocal.crate() != null; + if (result) { + parserLocal.crate().accept(visitor); + } + return result; + } + + public List blocks() { + return info.blocks(); + } + + public List semanticRegions() { + return info.semanticRegions(); + } + + public List structureItems() { + return info.structureItems(); + } + + public RustParseInfo info() { + return info; + } + + @Override + public List getDiagnostics() { + return info.errors(); + } + + @Override + protected synchronized void invalidate() { + info.clear(); + parser = null; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrSemanticAnalyzer.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrSemanticAnalyzer.java new file mode 100644 index 0000000..b98ecfc --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrSemanticAnalyzer.java @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2017 drrb + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.netbeans.modules.csl.api.ColoringAttributes; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.api.SemanticAnalyzer; +import org.netbeans.modules.parsing.spi.Scheduler; +import org.netbeans.modules.parsing.spi.SchedulerEvent; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + + +/** + * + */ +public class RustAntlrSemanticAnalyzer extends SemanticAnalyzer { +// private static final Logger LOG = Logger.getLogger(RustAntlrSemanticAnalyzer.class.getName()); + + private final Map> highlights = new HashMap<>(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + + @Override + public void run(RustAntlrParserResult result, SchedulerEvent event) { + try { + highlights.clear(); + cancelled.set(false); + for (RustSourceRegion region : result.info().semanticRegions()) { + System.out.println("highlight " + region); + highlights.put(region.range(), region.attributes()); + } + } catch (Exception e) { + e.printStackTrace(System.err); + if (e instanceof RuntimeException) { + throw ((RuntimeException) e); + } else { + throw new RuntimeException(e); + } + } + } + + @Override + public Map> getHighlights() { + return new HashMap<>(highlights); + } + + @Override + public int getPriority() { + return 0; + } + + @Override + public Class getSchedulerClass() { + return Scheduler.EDITOR_SENSITIVE_TASK_SCHEDULER; + } + + @Override + public void cancel() { + cancelled.set(true); + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrStructureScanner.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrStructureScanner.java new file mode 100644 index 0000000..98a1e70 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrStructureScanner.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParserResult; +import static com.github.drrb.rust.netbeans.parsing.antlr.RustFoldTypeProvider.BLOCKS; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.api.StructureItem; +import org.netbeans.modules.csl.api.StructureScanner; +import org.netbeans.modules.csl.spi.ParserResult; + +/** + * + * @author Tim Boudreau + */ +public class RustAntlrStructureScanner implements StructureScanner { + + @Override + public List scan(ParserResult pr) { + return scan((RustAntlrParserResult) pr); + } + + @Override + public Map> folds(ParserResult pr) { + return folds((RustAntlrParserResult) pr); + } + + @Override + public Configuration getConfiguration() { + return new Configuration( true, true, 4 ); + } + + private Map> folds(RustAntlrParserResult pr) { + // Need to return a mutable list here for sorting + return Collections.singletonMap(BLOCKS, new ArrayList<>(pr.blocks())); + } + + private List scan(RustAntlrParserResult pr) { + // Need to return a mutable list here for sorting + return new ArrayList<>(pr.structureItems()); + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustElementKind.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustElementKind.java new file mode 100644 index 0000000..d482f35 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustElementKind.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.netbeans.modules.csl.api.ElementKind; + +/** + * + * @author Tim Boudreau + */ +public enum RustElementKind { + + STRUCT, + TRAIT, + TYPE, + FUNCTION, + ENUM, + FIELD, + TYPE_REFERENCE, + LIFETIME, + ENUM_CONSTANT, + ATTR, + IMPL + ; + + public boolean isStructural() { + switch(this) { + case TRAIT : + case TYPE : + case ENUM : + case FIELD : + case FUNCTION : + case ENUM_CONSTANT : + case IMPL : + return true; + default : + return false; + } + } + + /** + * Provides a rough mapping to NetBeans' ElementKind enum + * (which has no items to distingush type vs structure, + * for example. + * + * @return + */ + public ElementKind toElementKind() { + switch (this) { + // An imperfect mapping to say the least + case TRAIT : + return ElementKind.ATTRIBUTE; + case STRUCT: + return ElementKind.INTERFACE; + case TYPE: + case ENUM: + return ElementKind.CLASS; + case FIELD: + case ENUM_CONSTANT: + return ElementKind.FIELD; + case FUNCTION: + return ElementKind.METHOD; + case LIFETIME: + return ElementKind.RULE; + case TYPE_REFERENCE : + return ElementKind.GLOBAL; + case ATTR : + return ElementKind.ATTRIBUTE; + case IMPL : + return ElementKind.CLASS; + default: + throw new AssertionError(this); + } + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustFoldTypeProvider.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustFoldTypeProvider.java new file mode 100644 index 0000000..8fd75eb --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustFoldTypeProvider.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.netbeans.RustLanguage; +import java.util.Arrays; +import java.util.Collection; +import org.netbeans.api.editor.fold.FoldTemplate; +import org.netbeans.api.editor.fold.FoldType; +import org.netbeans.api.editor.mimelookup.MimeRegistration; +import org.netbeans.api.editor.mimelookup.MimeRegistrations; +import org.netbeans.spi.editor.fold.FoldTypeProvider; + +/** + * + * @author Tim Boudreau + */ +@MimeRegistrations({ + @MimeRegistration( + mimeType = RustLanguage.MIME_TYPE, + service = FoldTypeProvider.class, + position = 1488)}) +public class RustFoldTypeProvider implements FoldTypeProvider { + + public static final String COMMENTS = "comments"; + public static final String BLOCKS = "blocks"; + private static final FoldType BLOCK_FOLDS = FoldType.create(BLOCKS, BLOCKS, FoldTemplate.DEFAULT_BLOCK ); + private static final FoldType COMMENT_FOLDS = FoldType.create(COMMENTS, COMMENTS, FoldTemplate.DEFAULT_BLOCK ); + + @Override + @SuppressWarnings("unchecked") + public Collection getValues(Class type) { + return Arrays.asList( BLOCK_FOLDS, COMMENT_FOLDS ); + } + + @Override + public boolean inheritable() { + return true; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustParseInfo.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustParseInfo.java new file mode 100644 index 0000000..5817bd4 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustParseInfo.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustParser; +import com.github.drrb.rust.netbeans.parsing.antlr.RustVisibility; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.OffsetRange; + +/** + * + * @author Tim Boudreau + */ +final class RustParseInfo { + + Set errors = new LinkedHashSet<>(); + Set blocks = new LinkedHashSet<>(); + private List structureItems = new ArrayList<>(); + private RustStructureItemImpl currStructureItem; + private final List semanticRegions = new LinkedList<>(); + private static final Set VISIBILITY_NA = EnumSet.noneOf(RustVisibility.class); + + void clear() { + errors.clear(); + blocks.clear(); + structureItems.clear(); + semanticRegions.clear(); + } + + void addSemanticRegion(RustElementKind kind, OffsetRange range) { + addSemanticRegion("", kind, range, false, VISIBILITY_NA, false); + } + + void addSemanticRegion(String name, RustElementKind kind, OffsetRange range, boolean mutable, Set visibility, boolean statyc) { + RustElementKind childOf = currStructureItem == null ? null : currStructureItem.kind; + semanticRegions.add(new SemanticRegion(kind, name, range, mutable, visibility, statyc || (kind == RustElementKind.FUNCTION && currStructureItem == null), childOf)); + } + + List semanticRegions() { + return semanticRegions; + } + + List structureItems() { + return structureItems; + } + + boolean hasSyntaxError; + + void addError(Error error) { + if (hasSyntaxError && !(error instanceof SyntaxError)) { + // not interested in more subtle errors from ErrNode if + // we already know the source has syntax errors + return; + } + hasSyntaxError |= error instanceof SyntaxError; + errors.add(error); + } + + void addStructureItem(RustStructureItemImpl item) { + List items = this.structureItems; + if (currStructureItem != null) { + items = currStructureItem.nested(); + item.setIn(currStructureItem.qName()); + } + items.add(item); + } + + void pushStructureItem(RustStructureItemImpl item, Runnable run) { + RustStructureItemImpl prev = currStructureItem; + addStructureItem(item); + currStructureItem = item; + try { + run.run(); + } finally { + currStructureItem = prev; + } + } + + void addBlock(RustParser.BlockContext ctx) { + blocks.add(new OffsetRange(ctx.getSourceInterval().a, ctx.getSourceInterval().b)); + } + + public List errors() { + return new ArrayList<>(errors); + } + + public List blocks() { + return new ArrayList<>(blocks); + } + + public String toString() { + StringBuilder sb = new StringBuilder("errors: ["); + for (Iterator it = errors.iterator(); it.hasNext();) { + Error err = it.next(); + sb.append(err.getDescription()).append('@').append(err.getStartPosition()) + .append(':').append(err.getEndPosition()); + if (it.hasNext()) { + sb.append(','); + } + } + sb.append("] structure: ["); + for (Iterator it = structureItems.iterator(); it.hasNext();) { + sb.append(it.next()); + if (it.hasNext()) { + sb.append(','); + } + } + sb.append("] blocks ["); + for (Iterator it = blocks.iterator(); it.hasNext();) { + OffsetRange range = it.next(); + sb.append(range.getStart()).append(":").append(range.getEnd()); + if (it.hasNext()) { + sb.append(','); + } + } + sb.append("] semantic ["); + for (Iterator it = semanticRegions.iterator(); it.hasNext();) { + sb.append(it.next()); + if (it.hasNext()) { + sb.append(','); + } + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustSourceRegion.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustSourceRegion.java new file mode 100644 index 0000000..846c2ef --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustSourceRegion.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import java.util.Optional; +import java.util.Set; +import org.netbeans.modules.csl.api.ColoringAttributes; +import org.netbeans.modules.csl.api.OffsetRange; + +/** + * + * @author Tim Boudreau + */ +public interface RustSourceRegion { + + Set attributes(); + + Optional childOf(); + + boolean hasVisibility(RustVisibility vis); + + boolean isMutable(); + + boolean isStatic(); + + RustElementKind kind(); + + OffsetRange range(); + + Set visibility(); + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItem.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItem.java new file mode 100644 index 0000000..7be4bca --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItem.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import java.util.List; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.api.StructureItem; + +/** + * + * @author Tim Boudreau + */ +public interface RustStructureItem extends StructureItem { + RustElementKind rustKind(); + + @Override + public List getNestedItems(); + + public OffsetRange range(); +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItemImpl.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItemImpl.java new file mode 100644 index 0000000..72f33e7 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItemImpl.java @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.netbeans.RustLanguage; +import static com.github.drrb.rust.netbeans.parsing.antlr.AntlrUtils.toOffsetRange; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import javax.swing.ImageIcon; +import org.antlr.v4.runtime.ParserRuleContext; +import org.netbeans.modules.csl.api.ElementHandle; +import org.netbeans.modules.csl.api.ElementKind; +import org.netbeans.modules.csl.api.HtmlFormatter; +import org.netbeans.modules.csl.api.Modifier; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.spi.ParserResult; +import org.netbeans.modules.parsing.api.Snapshot; +import org.openide.filesystems.FileObject; + +/** + * + * @author Tim Boudreau + */ +final class RustStructureItemImpl implements ElementHandle, RustStructureItem { + + private final String name; + final RustElementKind kind; + List nested; + private final FileObject file; + private final OffsetRange range; + private String in = ""; + + RustStructureItemImpl(String name, RustElementKind kind, Snapshot snapshot, ParserRuleContext ctx) { + this.name = name; + this.kind = kind; + this.file = snapshot.getSource().getFileObject(); + this.range = toOffsetRange(ctx); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name).append(" ").append(kind).append(" @").append(range.getStart()).append(":").append(range.getEnd()); + if (!in.isEmpty()) { + sb.append(" in " + in); + } + if (nested != null && !nested.isEmpty()) { + sb.append(" children: ["); + for (Iterator it = nested.iterator(); it.hasNext();) { + sb.append(it.next()); + if (it.hasNext()) { + sb.append(","); + } + } + sb.append("]"); + } + return sb.toString(); + } + + List nested() { + if (nested == null) { + nested = new ArrayList<>(5); + } + return nested; + } + + String qName() { + if (!in.isEmpty()) { + return in + "." + name; + } else { + return name; + } + } + + void setIn(String in) { + this.in = in; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getSortText() { + return getName(); + } + + @Override + public String getHtml(HtmlFormatter hf) { + return getName(); + } + + @Override + public ElementHandle getElementHandle() { + return this; + } + + public RustElementKind rustKind() { + return kind; + } + + @Override + public ElementKind getKind() { + return kind.toElementKind(); + } + + @Override + public Set getModifiers() { + return Collections.emptySet(); + } + + @Override + public boolean isLeaf() { + return nested == null; + } + + @Override + public List getNestedItems() { + if (nested == null) { + return Collections.emptyList(); + } + return nested; + } + + @Override + public long getPosition() { + return range.getStart(); + } + + @Override + public long getEndPosition() { + return range.getEnd(); + } + + public OffsetRange range() { + return range; + } + + @Override + public ImageIcon getCustomIcon() { + return null; + } + + @Override + public FileObject getFileObject() { + return this.file; + } + + @Override + public String getMimeType() { + return RustLanguage.MIME_TYPE; + } + + @Override + public String getIn() { + return in; + } + + @Override + public boolean signatureEquals(ElementHandle eh) { + return getName().equals(eh.getName()) && Objects.equals(eh.getIn(), getIn()); + } + + @Override + public OffsetRange getOffsetRange(ParserResult pr) { + return range; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o instanceof RustStructureItemImpl) { + RustStructureItemImpl other = (RustStructureItemImpl) o; + return file.equals(other.file) && qName().equals(other.qName()) && range.getStart() == other.range.getStart() && range.getEnd() == other.range.getEnd(); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(kind, name, file, range.getStart(), range.getEnd()); + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustVisibility.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustVisibility.java new file mode 100644 index 0000000..f54c990 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustVisibility.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.antlr.v4.runtime.Token; + +/** + * + * @author Tim Boudreau + */ + enum RustVisibility { + PUB("pub"), CRATE("crate"), SUPER("super"), IN("in"); + private final String stringValue; + + RustVisibility(String stringValue) { + this.stringValue = stringValue; + } + + public String toString() { + return stringValue; + } + + public static RustVisibility forToken(Token tk) { + switch (tk.getText()) { + case "pub": + return PUB; + case "crate": + return CRATE; + case "super": + return SUPER; + case "in": + return IN; + default: + return null; + } + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SemanticRegion.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SemanticRegion.java new file mode 100644 index 0000000..f8ac988 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SemanticRegion.java @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.netbeans.parsing.antlr.RustVisibility; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; +import org.netbeans.modules.csl.api.ColoringAttributes; +import org.netbeans.modules.csl.api.OffsetRange; + +/** + * + * @author Tim Boudreau + */ +final class SemanticRegion implements RustSourceRegion { + + public final RustElementKind kind; + public final String text; + public final OffsetRange range; + public final boolean mutable; + public final Set visibility; + public final boolean statyc; + private final RustElementKind childOf; + + SemanticRegion(RustElementKind kind, String text, OffsetRange range, boolean mutable, Set visibility, boolean statyc, RustElementKind childOf) { + assert range != null : "Range is null"; + assert kind != null : "Kind is null"; + this.kind = kind; + this.text = text; + this.range = range; + this.mutable = mutable; + this.visibility = visibility; + this.statyc = statyc; + this.childOf = childOf; + } + + public String toString() { + StringBuilder sb = new StringBuilder(kind.name()); + sb.append(" @").append(range.getStart()).append(':').append(range.getEnd()); + if (mutable) { + sb.append(" mut"); + } + if (statyc) { + sb.append(" static"); + } + if (!visibility.isEmpty()) { + sb.append(" visibility: ["); + for (Iterator it = visibility.iterator(); it.hasNext();) { + RustVisibility vis = it.next(); + sb.append(' ').append(vis); + } + sb.append(']'); + } + if (text != null && !text.isEmpty()) { + String txt = text.replaceAll("\t", "\\\\t").replaceAll("\n", "\\\\n"); + sb.append(" text: '").append(txt).append('\''); + } + if (childOf != null) { + sb.append (" under: ").append(childOf); + } + return sb.toString(); + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o instanceof SemanticRegion) { + SemanticRegion other = (SemanticRegion) o; + return range.getStart() == other.range().getStart() && range.getEnd() == other.range().getEnd(); + } + return false; + } + + @Override + public RustElementKind kind() { + return kind; + } + + @Override + public Optional childOf() { + return Optional.ofNullable(childOf); + } + + @Override + public OffsetRange range() { + return range; + } + + @Override + public boolean isMutable() { + return mutable; + } + + @Override + public Set visibility() { + return visibility; + } + + @Override + public boolean isStatic() { + return statyc; + } + + @Override + public boolean hasVisibility(RustVisibility vis) { + return visibility != null && visibility.contains(vis); + } + + @Override + public Set attributes() { + // Pending - differentiate things like trait methods from + // struct/type methods using childOf and different colorings + Set result = EnumSet.noneOf(ColoringAttributes.class); + if (mutable) { + result.add(ColoringAttributes.GLOBAL); //XXX + } + if (hasVisibility(RustVisibility.PUB)) { + result.add(ColoringAttributes.PUBLIC); + } + if (hasVisibility(RustVisibility.CRATE)) { + result.add(ColoringAttributes.PACKAGE_PRIVATE); + } + if (hasVisibility(RustVisibility.SUPER)) { + result.add(ColoringAttributes.PROTECTED); + } + if (hasVisibility(RustVisibility.IN)) { + result.add(ColoringAttributes.CUSTOM3); + } + if (statyc) { + result.add(ColoringAttributes.STATIC); + } + switch (kind) { + case ENUM_CONSTANT: + result.add(ColoringAttributes.ENUM); + break; + case FIELD: + result.add(ColoringAttributes.FIELD); + break; + case FUNCTION: + result.add(ColoringAttributes.METHOD); + break; + case ENUM: + result.add(ColoringAttributes.CLASS); + break; + case TYPE: + result.add(ColoringAttributes.CLASS); + break; + case STRUCT: + result.add(ColoringAttributes.CLASS); + break; + case TRAIT: + result.add(ColoringAttributes.INTERFACE); + break; + case LIFETIME: + result.add(ColoringAttributes.CUSTOM1); + break; + case TYPE_REFERENCE: + result.add(ColoringAttributes.TYPE_PARAMETER_USE); + break; + case ATTR: + result.add(ColoringAttributes.ANNOTATION_TYPE); + break; + case IMPL: + result.add(ColoringAttributes.CLASS); + break; + default: + throw new AssertionError(kind); + } + return result; + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SyntaxError.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SyntaxError.java new file mode 100644 index 0000000..9fa345b --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SyntaxError.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.antlr.v4.runtime.Token; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.Severity; +import org.openide.filesystems.FileObject; + +/** + * + * @author Tim Boudreau + */ +public class SyntaxError implements Error { + + private final AntlrTokenID type; + private final int startIndex; + private final int stopIndex; + private final int line; + private final int charPositionInLine; + private final int channel; + private final int tokenIndex; + private final String description; + private final FileObject fo; + private final String text; + + public SyntaxError(Token token, String description, FileObject fo) { + this.description = description; + this.type = CommonRustTokenIDs.forTokenType(token.getType()); + this.line = token.getLine(); + this.charPositionInLine = token.getCharPositionInLine(); + this.startIndex = token.getStartIndex(); + this.stopIndex = token.getStopIndex(); + this.channel = token.getChannel(); + this.tokenIndex = token.getTokenIndex(); + this.fo = fo; + this.text = token.getText(); + } + + public AntlrTokenID type() { + return type; + } + + @Override + public String toString() { + return "@line: " + line + ":" + charPositionInLine + + " (pos: " + startIndex + ":" + (stopIndex + 1) + ") tok " + + tokenIndex + " '" + text + "' (" + type + "): " + description; + } + + public int line() { + return line; + } + + public int charPositionInLine() { + return charPositionInLine; + } + + public int channel() { + return channel; + } + + public int tokenIndex() { + return tokenIndex; + } + + @Override + public String getDisplayName() { + return "Syntax error"; + } + + @Override + public String getDescription() { + return description; + } + + @Override + public String getKey() { + return type.name() + + ":" + line + + ":" + charPositionInLine + + ":" + fo.getName(); + } + + @Override + public FileObject getFile() { + return fo; + } + + @Override + public int getStartPosition() { + return startIndex; + } + + @Override + public int getEndPosition() { + return stopIndex + 1; + } + + @Override + public boolean isLineError() { + return true; + } + + @Override + public Severity getSeverity() { + return Severity.FATAL; + } + + @Override + public Object[] getParameters() { + return new Object[0]; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o instanceof SyntaxError) { + return getKey().equals(((SyntaxError) o).getKey()); + } + return false; + } + + @Override + public int hashCode() { + return ((line + 1) * (charPositionInLine * 7) + + (this.type.ordinal() + 1)) * + (51 * (fo.getName().hashCode() + 1)); + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/ParseUtil.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/TokenCategorizer.java similarity index 63% rename from src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/ParseUtil.java rename to src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/TokenCategorizer.java index b3bc29c..260c89f 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/ParseUtil.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/TokenCategorizer.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2017 drrb + * Copyright (C) 2018 Tim Boudreau * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software @@ -14,16 +14,14 @@ * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ -package com.github.drrb.rust.netbeans.parsing.javacc; +package com.github.drrb.rust.netbeans.parsing.antlr; -import org.netbeans.modules.csl.api.OffsetRange; - -public class ParseUtil { +/** + * + * @author Tim Boudreau + */ +public interface TokenCategorizer { - private ParseUtil() { - } + public String categoryFor(int tokenType, String displayName, String symbolicName, String literalName); - public static OffsetRange offsetRange(SimpleNode node) { - return new OffsetRange(node.jjtGetFirstToken().absoluteBeginPosition - 1, node.jjtGetLastToken().absoluteEndPosition - 1); - } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStruct.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStruct.java index 5c85752..5a24af7 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStruct.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStruct.java @@ -35,6 +35,10 @@ public class RustStruct { this.body = body; } + public String toString() { + return name + "@" + offsetRange.getStart() + ":" + offsetRange.getEnd() + " {" + body + "}"; + } + public String getName() { return name; } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructBody.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructBody.java index a9944b3..7648c46 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructBody.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructBody.java @@ -17,6 +17,7 @@ package com.github.drrb.rust.netbeans.parsing.index; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import org.netbeans.modules.csl.api.OffsetRange; @@ -31,7 +32,18 @@ public class RustStructBody { RustStructBody(OffsetRange offsetRange, List fields) { this.offsetRange = offsetRange; - this.fields = fields; + this.fields = Collections.unmodifiableList(fields); + } + + public String toString() { + StringBuilder sb= new StringBuilder(); + for (Iterator it=fields.iterator(); it.hasNext();) { + sb.append(it.next()); + if (it.hasNext()) { + sb.append(','); + } + } + return sb.toString(); } public OffsetRange getOffsetRange() { diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructField.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructField.java index 96fb731..dfcac3e 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructField.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructField.java @@ -38,4 +38,8 @@ public String getName() { public OffsetRange getOffsetRange() { return offsetRange; } + + public String toString() { + return name + "@" + offsetRange.getStart() + ":" + offsetRange.getEnd(); + } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/JavaccCharStream.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/JavaccCharStream.java deleted file mode 100644 index 0ceeb0c..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/JavaccCharStream.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing.javacc; - -import org.netbeans.spi.lexer.LexerInput; - -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.io.UnsupportedEncodingException; - -public class JavaccCharStream implements CharStream { - - private LexerInput input; - - public int offset = 0; - private int tokenOffset = 0; - private boolean trackLineColumn = true; - - public JavaccCharStream(LexerInput input) { - this.input = input; - } - - public char BeginToken() throws IOException { - tokenOffset = 0; - return readChar(); - } - - public String GetImage() { - return input.readText(input.readLength() - tokenOffset, input.readLength()).toString(); - } - - public char[] GetSuffix(int len) { - if (len > input.readLength()) - throw new IllegalArgumentException(); - return input.readText(input.readLength() - len, input.readLength()).toString().toCharArray(); - } - - public void ReInit(Reader stream, int i, int i0) { - throw new UnsupportedOperationException("Not yet implemented"); - } - - public void ReInit(InputStream stream, String encoding, int i, int i0) throws UnsupportedEncodingException { - throw new UnsupportedOperationException("Not yet implemented"); - } - - public void backup(int i) { - offset -= i; - tokenOffset -= i; - tokenOffset = tokenOffset < 0 ? 0 : tokenOffset; - input.backup(i); - } - - public int getBeginColumn() { - return 0; - } - - public int getBeginLine() { - return 0; - } - - public int getEndColumn() { - return 0; - } - - public int getEndLine() { - return 0; - } - - public char readChar() throws IOException { - offset++; - tokenOffset++; - int result = input.read(); - if (result == LexerInput.EOF) { - if (tokenOffset > 1) { //todo: why? - backup(1); - } - throw new IOException("LexerInput EOF"); - } - return (char) result; - } - - @Override - public int getColumn() { - return 0; - } - - @Override - public int getLine() { - return 0; - } - - @Override - public void Done() { - - } - - @Override - public void setTabSize(int i) { - - } - - @Override - public int getTabSize() { - return 0; - } - - @Override - public boolean getTrackLineColumn() { - return trackLineColumn; - } - - @Override - public void setTrackLineColumn(boolean trackLineColumn) { - this.trackLineColumn = trackLineColumn; - } -} - diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustToken.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustToken.java index 8884d9d..a06e156 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustToken.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustToken.java @@ -16,96 +16,138 @@ */ package com.github.drrb.rust.netbeans.parsing.javacc; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; import org.netbeans.modules.csl.api.OffsetRange; -import java.util.LinkedList; -import java.util.List; -import static com.github.drrb.rust.netbeans.parsing.RustTokenId.EOF; - -public class RustToken extends Token { - private final RustTokenId enumKind; - - public RustToken(int kind, String image) { +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; +import java.util.Objects; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.TokenSource; +import org.netbeans.api.lexer.PartType; +import org.netbeans.api.lexer.Token; +import org.netbeans.api.lexer.TokenHierarchy; + +public class RustToken extends Token implements org.antlr.v4.runtime.Token { + private final AntlrTokenID enumKind; + private int kind; + private String image; + private final OffsetRange range; + public RustToken(int kind, String image, OffsetRange range) { this.kind = kind; - this.enumKind = RustTokenId.get(kind); + this.enumKind = CommonRustTokenIDs.forTokenType(kind); this.image = image; + this.range = range; } public boolean isEof() { - return enumKind == EOF; + return enumKind == CommonRustTokenIDs.eof(); } - public RustTokenId kind() { + public AntlrTokenID kind() { return enumKind; } - public RustTokenId id() { + public AntlrTokenID id() { return kind(); } - public RustToken specialToken() { - return (RustToken) specialToken; + @Override + public String toString() { + return enumKind + ": '" + image + "'"; + } + + public OffsetRange offsetRange() { + return range; + } + + @Override + public CharSequence text() { + return image; } - public boolean hasSpecialToken() { - return specialToken != null; + @Override + public boolean isCustomText() { + return !Objects.equals(image, enumKind.literalName()); } - public boolean hasNext() { - return next != null; + @Override + public int length() { + return range.getLength(); } - public RustToken next() { - return (RustToken) next; + @Override + public int offset(TokenHierarchy th) { + return range.getStart(); } - public boolean hasNextSpecialToken() { - return hasNext() && next().hasSpecialToken(); + @Override + public boolean isFlyweight() { + return false; } - public RustToken nextSpecialToken() { - return next().getEarliestSpecialToken(); + @Override + public PartType partType() { + return PartType.COMPLETE; } - public RustToken getEarliestSpecialToken() { - if (specialToken == null) { - return null; - } - Token token = this; - while (token.specialToken != null) { - token = token.specialToken; - } - return (RustToken) token; + @Override + public boolean hasProperties() { + return false; } @Override - public String toString() { - return enumKind + ": '" + image + "'"; + public Object getProperty(Object o) { + return null; } - public RustToken nextTokenMaybeSpecial() { - if (hasNextSpecialToken()) { - return nextSpecialToken(); - } else if (hasNext()) { - return next(); - } else { - return null; - } + @Override + public String getText() { + return image; } - public List withSpecialTokens() { - LinkedList thisWithSpecialTokens = new LinkedList<>(); - RustToken token = this; - do { - thisWithSpecialTokens.addFirst(token); - token = token.specialToken(); - } while (token != null); - return thisWithSpecialTokens; + @Override + public int getType() { + return kind; } - public OffsetRange offsetRange() { - return new OffsetRange(absoluteBeginPosition - 1, absoluteEndPosition - 1); + @Override + public int getLine() { + return 0; + } + + @Override + public int getCharPositionInLine() { + return 0; + } + + @Override + public int getChannel() { + return 0; + } + + @Override + public int getTokenIndex() { + return 0; + } + + @Override + public int getStartIndex() { + return range.getStart(); + } + + @Override + public int getStopIndex() { + return range.getEnd() -1; + } + + @Override + public TokenSource getTokenSource() { + return null; + } + + @Override + public CharStream getInputStream() { + return null; } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustTokenFactory.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustTokenFactory.java deleted file mode 100644 index 95fec8c..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustTokenFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing.javacc; - -public class RustTokenFactory { - - public static Token newToken(int ofKind, String tokenImage) { - return new RustToken(maybeTranslateSubkind(ofKind), tokenImage); - } - - private static int maybeTranslateSubkind(int kind) { - switch(kind) { - case RustParserConstants.RAW_STRING_LITERAL_0: - case RustParserConstants.RAW_STRING_LITERAL_1: - case RustParserConstants.RAW_STRING_LITERAL_2: - case RustParserConstants.RAW_STRING_LITERAL_3: - return RustParserConstants.RAW_STRING_LITERAL; - case RustParserConstants.RAW_BYTE_STRING_LITERAL_0: - case RustParserConstants.RAW_BYTE_STRING_LITERAL_1: - case RustParserConstants.RAW_BYTE_STRING_LITERAL_2: - case RustParserConstants.RAW_BYTE_STRING_LITERAL_3: - return RustParserConstants.RAW_BYTE_STRING_LITERAL; - default: - return kind; - } - } -} - diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/SimpleCharStream.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/SimpleCharStream.java deleted file mode 100644 index 2e24ea7..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/SimpleCharStream.java +++ /dev/null @@ -1,500 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 7.0 */ -/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ -package com.github.drrb.rust.netbeans.parsing.javacc; - -/** - * An implementation of interface CharStream, where the stream is assumed to - * contain only ASCII characters (without unicode processing). - */ - -public class SimpleCharStream implements CharStream -{ - protected int totalCharsRead = 0; - protected int absoluteTokenBegin = 0; - public final int getAbsoluteTokenBegin() { - return absoluteTokenBegin; - } - -/** Whether parser is static. */ - public static final boolean staticFlag = false; - int bufsize; - int available; - int tokenBegin; -/** Position in buffer. */ - public int bufpos = -1; - protected int bufline[]; - protected int bufcolumn[]; - - protected int column = 0; - protected int line = 1; - - protected boolean prevCharIsCR = false; - protected boolean prevCharIsLF = false; - - protected java.io.Reader inputStream; - - protected char[] buffer; - protected int maxNextCharInd = 0; - protected int inBuf = 0; - protected int tabSize = 1; - protected boolean trackLineColumn = true; - - public void setTabSize(int i) { tabSize = i; } - public int getTabSize() { return tabSize; } - - - - protected void ExpandBuff(boolean wrapAround) - { - char[] newbuffer = new char[bufsize + 2048]; - int newbufline[] = new int[bufsize + 2048]; - int newbufcolumn[] = new int[bufsize + 2048]; - - try - { - if (wrapAround) - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos); - buffer = newbuffer; - - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); - bufline = newbufline; - - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); - bufcolumn = newbufcolumn; - - maxNextCharInd = (bufpos += (bufsize - tokenBegin)); - } - else - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - buffer = newbuffer; - - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - bufline = newbufline; - - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - bufcolumn = newbufcolumn; - - maxNextCharInd = (bufpos -= tokenBegin); - } - } - catch (Throwable t) - { - throw new Error(t.getMessage()); - } - - - bufsize += 2048; - available = bufsize; - tokenBegin = 0; - } - - protected void FillBuff() throws java.io.IOException - { - if (maxNextCharInd == available) - { - if (available == bufsize) - { - if (tokenBegin > 2048) - { - bufpos = maxNextCharInd = 0; - available = tokenBegin; - } - else if (tokenBegin < 0) - bufpos = maxNextCharInd = 0; - else - ExpandBuff(false); - } - else if (available > tokenBegin) - available = bufsize; - else if ((tokenBegin - available) < 2048) - ExpandBuff(true); - else - available = tokenBegin; - } - - int i; - try { - if ((i = inputStream.read(buffer, maxNextCharInd, available - maxNextCharInd)) == -1) - { - inputStream.close(); - throw new java.io.IOException(); - } - else - maxNextCharInd += i; - return; - } - catch(java.io.IOException e) { - --bufpos; - backup(0); - if (tokenBegin == -1) - tokenBegin = bufpos; - throw e; - } - } - -/** Start. */ - public char BeginToken() throws java.io.IOException - { - tokenBegin = -1; - char c = readChar(); - tokenBegin = bufpos; - absoluteTokenBegin = totalCharsRead; - - return c; - } - - protected void UpdateLineColumn(char c) - { - column++; - - if (prevCharIsLF) - { - prevCharIsLF = false; - line += (column = 1); - } - else if (prevCharIsCR) - { - prevCharIsCR = false; - if (c == '\n') - { - prevCharIsLF = true; - } - else - line += (column = 1); - } - - switch (c) - { - case '\r' : - prevCharIsCR = true; - break; - case '\n' : - prevCharIsLF = true; - break; - case '\t' : - column--; - column += (tabSize - (column % tabSize)); - break; - default : - break; - } - - bufline[bufpos] = line; - bufcolumn[bufpos] = column; - } - -/** Read a character. */ - public char readChar() throws java.io.IOException - { - if (inBuf > 0) - { - --inBuf; - - if (++bufpos == bufsize) - bufpos = 0; - - totalCharsRead++; - return buffer[bufpos]; - } - - if (++bufpos >= maxNextCharInd) - FillBuff(); - - totalCharsRead++; - char c = buffer[bufpos]; - - UpdateLineColumn(c); - return c; - } - - @Deprecated - /** - * @deprecated - * @see #getEndColumn - */ - - public int getColumn() { - return bufcolumn[bufpos]; - } - - @Deprecated - /** - * @deprecated - * @see #getEndLine - */ - - public int getLine() { - return bufline[bufpos]; - } - - /** Get token end column number. */ - public int getEndColumn() { - return bufcolumn[bufpos]; - } - - /** Get token end line number. */ - public int getEndLine() { - return bufline[bufpos]; - } - - /** Get token beginning column number. */ - public int getBeginColumn() { - return bufcolumn[tokenBegin]; - } - - /** Get token beginning line number. */ - public int getBeginLine() { - return bufline[tokenBegin]; - } - -/** Backup a number of characters. */ - public void backup(int amount) { - - inBuf += amount; - totalCharsRead -= amount; - if ((bufpos -= amount) < 0) - bufpos += bufsize; - } - - /** Constructor. */ - public SimpleCharStream(java.io.Reader dstream, int startline, - int startcolumn, int buffersize) - { - inputStream = dstream; - line = startline; - column = startcolumn - 1; - - available = bufsize = buffersize; - buffer = new char[buffersize]; - bufline = new int[buffersize]; - bufcolumn = new int[buffersize]; - } - - /** Constructor. */ - public SimpleCharStream(java.io.Reader dstream, int startline, - int startcolumn) - { - this(dstream, startline, startcolumn, 4096); - } - - /** Constructor. */ - public SimpleCharStream(java.io.Reader dstream) - { - this(dstream, 1, 1, 4096); - } - - /** Reinitialise. */ - public void ReInit(java.io.Reader dstream, int startline, - int startcolumn, int buffersize) - { - inputStream = dstream; - line = startline; - column = startcolumn - 1; - - if (buffer == null || buffersize != buffer.length) - { - available = bufsize = buffersize; - buffer = new char[buffersize]; - bufline = new int[buffersize]; - bufcolumn = new int[buffersize]; - } - prevCharIsLF = prevCharIsCR = false; - tokenBegin = inBuf = maxNextCharInd = 0; - bufpos = -1; - } - - /** Reinitialise. */ - public void ReInit(java.io.Reader dstream, int startline, - int startcolumn) - { - ReInit(dstream, startline, startcolumn, 4096); - } - - /** Reinitialise. */ - public void ReInit(java.io.Reader dstream) - { - ReInit(dstream, 1, 1, 4096); - } - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline, - int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException - { - this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, int startline, - int startcolumn, int buffersize) - { - this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline, - int startcolumn) throws java.io.UnsupportedEncodingException - { - this(dstream, encoding, startline, startcolumn, 4096); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, int startline, - int startcolumn) - { - this(dstream, startline, startcolumn, 4096); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException - { - this(dstream, encoding, 1, 1, 4096); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream) - { - this(dstream, 1, 1, 4096); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding, int startline, - int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException - { - ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, int startline, - int startcolumn, int buffersize) - { - ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException - { - ReInit(dstream, encoding, 1, 1, 4096); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream) - { - ReInit(dstream, 1, 1, 4096); - } - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding, int startline, - int startcolumn) throws java.io.UnsupportedEncodingException - { - ReInit(dstream, encoding, startline, startcolumn, 4096); - } - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, int startline, - int startcolumn) - { - ReInit(dstream, startline, startcolumn, 4096); - } - /** Get token literal value. */ - public String GetImage() - { - if (bufpos >= tokenBegin) - return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); - else - return new String(buffer, tokenBegin, bufsize - tokenBegin) + - new String(buffer, 0, bufpos + 1); - } - - /** Get the suffix. */ - public char[] GetSuffix(int len) - { - char[] ret = new char[len]; - - if ((bufpos + 1) >= len) - System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); - else - { - System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, - len - bufpos - 1); - System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); - } - - return ret; - } - - /** Reset buffer when finished. */ - public void Done() - { - buffer = null; - bufline = null; - bufcolumn = null; - } - - /** - * Method to adjust line and column numbers for the start of a token. - */ - public void adjustBeginLineColumn(int newLine, int newCol) - { - int start = tokenBegin; - int len; - - if (bufpos >= tokenBegin) - { - len = bufpos - tokenBegin + inBuf + 1; - } - else - { - len = bufsize - tokenBegin + bufpos + 1 + inBuf; - } - - int i = 0, j = 0, k = 0; - int nextColDiff = 0, columnDiff = 0; - - while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) - { - bufline[j] = newLine; - nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j]; - bufcolumn[j] = newCol + columnDiff; - columnDiff = nextColDiff; - i++; - } - - if (i < len) - { - bufline[j] = newLine++; - bufcolumn[j] = newCol + columnDiff; - - while (i++ < len) - { - if (bufline[j = start % bufsize] != bufline[++start % bufsize]) - bufline[j] = newLine++; - else - bufline[j] = newLine; - } - } - - line = bufline[j]; - column = bufcolumn[j]; - } - public boolean getTrackLineColumn() { return trackLineColumn; } - public void setTrackLineColumn(boolean tlc) { trackLineColumn = tlc; } -} -/* JavaCC - OriginalChecksum=b094525db5a9a7246ced7b3d7fc11c7f (do not edit this line) */ diff --git a/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustLexer.java b/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustLexer.java index cb4ab56..7f3380a 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustLexer.java +++ b/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustLexer.java @@ -16,7 +16,7 @@ */ package com.github.drrb.rust.netbeans.rustbridge; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; /** * @@ -28,7 +28,7 @@ public class RustLexer { @Override public RustToken.ByValue nextToken() { RustToken.ByValue token = new RustToken.ByValue(); - token.type = RustTokenId.EOF.ordinal(); + token.type = CommonRustTokenIDs.eof().ordinal(); return token; } diff --git a/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustToken.java b/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustToken.java index d6c4493..344e210 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustToken.java +++ b/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustToken.java @@ -16,10 +16,12 @@ */ package com.github.drrb.rust.netbeans.rustbridge; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; import com.sun.jna.Structure; import static java.util.Arrays.asList; import java.util.List; +import org.antlr.v4.runtime.Token; /** * @@ -39,12 +41,22 @@ public static class ByValue extends RustToken implements Structure.ByValue { public int endChar; public int type; + public static RustToken of(Token tok) { + RustToken result = new RustToken(); + result.startLine = tok.getLine(); + result.startCol = tok.getCharPositionInLine(); + result.type = tok.getType(); + result.endCol = tok.getCharPositionInLine() + ((tok.getStopIndex() + 1) - tok.getStartIndex()); + result.endLine = tok.getLine(); + return result; + } + boolean isEof() { - return getType() == RustTokenId.EOF; + return getType() == CommonRustTokenIDs.eof(); } - public RustTokenId getType() { - return RustTokenId.values()[type]; + public AntlrTokenID getType() { + return CommonRustTokenIDs.forTokenType(type); } public int length() { diff --git a/src/main/resources/com/github/drrb/rust/netbeans/FontAndColors.xml b/src/main/resources/com/github/drrb/rust/netbeans/FontAndColors.xml index a6b7392..81a5037 100644 --- a/src/main/resources/com/github/drrb/rust/netbeans/FontAndColors.xml +++ b/src/main/resources/com/github/drrb/rust/netbeans/FontAndColors.xml @@ -28,8 +28,10 @@ + + - + diff --git a/src/test/data/index/project/struct/src/main.rs b/src/test/data/index/project/struct/src/main.rs index 54f3fc4..92cf826 100644 --- a/src/test/data/index/project/struct/src/main.rs +++ b/src/test/data/index/project/struct/src/main.rs @@ -1,4 +1,4 @@ struct Person { name: String, age: usize, -} \ No newline at end of file +} diff --git a/src/test/data/parse/errors/errors_in_items.rs.errors b/src/test/data/parse/errors/errors_in_items.rs.errors index 196dd27..22b8206 100644 --- a/src/test/data/parse/errors/errors_in_items.rs.errors +++ b/src/test/data/parse/errors/errors_in_items.rs.errors @@ -1,104 +1,6 @@ -[rust.parse.message] 31-32:Parse error ; Encountered " "}" "} "" at line 3, column 1. -Was expecting one of: - ... - ... - ... - ... - ... - ... - ... - "::" ... - "(" ... - "-" ... - "*" ... - "&" ... - "!" ... - "false" ... - "for" ... - "if" ... - "loop" ... - "return" ... - "true" ... - "while" ... - ... -