From 8b7984c77905d273365bc4adc09c315e35d84893 Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Tue, 13 Sep 2022 19:31:28 -0500 Subject: [PATCH 01/20] Fixing minimum CXX standard required Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- runtime/Cpp/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/Cpp/README.md b/runtime/Cpp/README.md index 622289ba77..fb5b22da4e 100644 --- a/runtime/Cpp/README.md +++ b/runtime/Cpp/README.md @@ -35,12 +35,12 @@ The C++ target has been the work of the following people: ### Build + Usage Notes -The minimum C++ version to compile the ANTLR C++ runtime with is C++11. The supplied projects can built the runtime either as static or dynamic library, as both 32bit and 64bit arch. The macOS project contains a target for iOS and can also be built using cmake (instead of XCode). +The minimum C++ version to compile the ANTLR C++ runtime with is C++17. The supplied projects can built the runtime either as static or dynamic library, as both 32bit and 64bit arch. The macOS project contains a target for iOS and can also be built using cmake (instead of XCode). Include the antlr4-runtime.h umbrella header in your target application to get everything needed to use the library. If you are compiling with cmake, the minimum version required is cmake 2.8. -By default, the libraries produced by the CMake build target C++11. If you want to target a different C++ standard, you can explicitly pass the standard - e.g. `-DCMAKE_CXX_STANDARD=17`. +By default, the libraries produced by the CMake build target C++17. If you want to target a different C++ standard, you can explicitly pass the standard - e.g. `-DCMAKE_CXX_STANDARD=17`. #### Compiling on Windows with Visual Studio using he Visual Studio projects Simply open the VS project from the runtime folder (VS 2019+) and build it. From 531654ed47955a68c85c4feddf045176a97ac48c Mon Sep 17 00:00:00 2001 From: "Jim.Idle" Date: Wed, 7 Sep 2022 11:55:17 +0800 Subject: [PATCH 02/20] fix: Restore missing changes to v4 of go runtime I had not though about this, but I guess some of the fix PRs did not have one fix that was in the legacy version of the runtime, copied in to the v4 branch. This commit fixes that. Signed-off-by: Jim.Idle Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- runtime/Go/antlr/antlrdoc.go | 2 +- runtime/Go/antlr/v4/prediction_context.go | 42 ++++++++++++++++++----- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/runtime/Go/antlr/antlrdoc.go b/runtime/Go/antlr/antlrdoc.go index 16b55514a2..b34f3e91a5 100644 --- a/runtime/Go/antlr/antlrdoc.go +++ b/runtime/Go/antlr/antlrdoc.go @@ -48,7 +48,7 @@ And the generate.sh file will look similar to this: #!/bin/sh alias antlr4='java -Xmx500M -cp "./antlr4-4.11.1-complete.jar:$CLASSPATH" org.antlr.v4.Tool' - antlr4 -Dlanguage=Go -no-visitor -package tgram *.g4 + antlr4 -Dlanguage=Go -no-visitor -package parser *.g4 depending on whether you want visitors or listeners or any other ANTLR options. diff --git a/runtime/Go/antlr/v4/prediction_context.go b/runtime/Go/antlr/v4/prediction_context.go index 72d24c3260..4fcad69a9c 100644 --- a/runtime/Go/antlr/v4/prediction_context.go +++ b/runtime/Go/antlr/v4/prediction_context.go @@ -376,11 +376,20 @@ func predictionContextFromRuleContext(a *ATN, outerContext RuleContext) Predicti } func merge(a, b PredictionContext, rootIsWildcard bool, mergeCache *DoubleDict) PredictionContext { - // share same graph if both same - if a == b { + + // Share same graph if both same + // + if a == b || a.Equals(b) { return a } + // In Java, EmptyPredictionContext inherits from SingletonPredictionContext, and so the test + // in java for SingletonPredictionContext will succeed and a new ArrayPredictionContext will be created + // from it. + // In go, EmptyPredictionContext does not equate to SingletonPredictionContext and so that conversion + // will fail. We need to test for both Empty and Singleton and create an ArrayPredictionContext from + // either of them. + ac, ok1 := a.(*BaseSingletonPredictionContext) bc, ok2 := b.(*BaseSingletonPredictionContext) @@ -397,14 +406,30 @@ func merge(a, b PredictionContext, rootIsWildcard bool, mergeCache *DoubleDict) return b } } - // convert singleton so both are arrays to normalize - if _, ok := a.(*BaseSingletonPredictionContext); ok { - a = NewArrayPredictionContext([]PredictionContext{a.GetParent(0)}, []int{a.getReturnState(0)}) + + // Convert Singleton or Empty so both are arrays to normalize - We should not use the existing parameters + // here. + // + // TODO: I think that maybe the Prediction Context structs should be redone as there is a chance we will see this mess again - maybe redo the logic here + + var arp, arb *ArrayPredictionContext + var ok bool + if arp, ok = a.(*ArrayPredictionContext); ok { + } else if _, ok = a.(*BaseSingletonPredictionContext); ok { + arp = NewArrayPredictionContext([]PredictionContext{a.GetParent(0)}, []int{a.getReturnState(0)}) + } else if _, ok = a.(*EmptyPredictionContext); ok { + arp = NewArrayPredictionContext([]PredictionContext{}, []int{}) } - if _, ok := b.(*BaseSingletonPredictionContext); ok { - b = NewArrayPredictionContext([]PredictionContext{b.GetParent(0)}, []int{b.getReturnState(0)}) + + if arb, ok = b.(*ArrayPredictionContext); ok { + } else if _, ok = b.(*BaseSingletonPredictionContext); ok { + arb = NewArrayPredictionContext([]PredictionContext{b.GetParent(0)}, []int{b.getReturnState(0)}) + } else if _, ok = b.(*EmptyPredictionContext); ok { + arb = NewArrayPredictionContext([]PredictionContext{}, []int{}) } - return mergeArrays(a.(*ArrayPredictionContext), b.(*ArrayPredictionContext), rootIsWildcard, mergeCache) + + // Both arp and arb + return mergeArrays(arp, arb, rootIsWildcard, mergeCache) } // Merge two {@link SingletonBasePredictionContext} instances. @@ -677,6 +702,7 @@ func mergeArrays(a, b *ArrayPredictionContext, rootIsWildcard bool, mergeCache * // if we created same array as a or b, return that instead // TODO: track whether this is possible above during merge sort for speed + // TODO: In go, I do not think we can just do M == xx as M is a brand new allocation. This could be causing allocation problems if M == a { if mergeCache != nil { mergeCache.set(a.Hash(), b.Hash(), a) From 9e96de873106c8f607e05dfc0b392742409ebd24 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sun, 11 Sep 2022 11:15:39 -0700 Subject: [PATCH 03/20] tweak spelling errors in Interpreter Signed-off-by: Terence Parr Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/gui/Interpreter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/src/org/antlr/v4/gui/Interpreter.java b/tool/src/org/antlr/v4/gui/Interpreter.java index 0e27cd13e6..8ef804d147 100644 --- a/tool/src/org/antlr/v4/gui/Interpreter.java +++ b/tool/src/org/antlr/v4/gui/Interpreter.java @@ -19,7 +19,7 @@ /** Interpret a lexer/parser, optionally printing tree string and dumping profile info * - * $ java org.antlr.v4.runtime.misc.Intrepreter [X.g4|XParser.g4 XLexer.g4] startRuleName inputFileName + * $ java org.antlr.v4.gui.Interpreter [X.g4|XParser.g4 XLexer.g4] startRuleName inputFileName * [-tree] * [-gui] * [-trace] @@ -62,7 +62,7 @@ public void importTokensFromTokensFile() { public Interpreter(String[] args) throws Exception { if ( args.length < 2 ) { - System.err.println("java org.antlr.v4.guIntrepreter [X.g4|XParser.g4 XLexer.g4] startRuleName\n" + + System.err.println("java org.antlr.v4.gui.Intrepreter [X.g4|XParser.g4 XLexer.g4] startRuleName\n" + " [-tokens] [-tree] [-gui] [-encoding encodingname]\n" + " [-trace] [-profile filename.csv] [input-filename(s)]"); System.err.println("Omitting input-filename makes rig read from stdin."); From 3dacdff458e541e4b9eea9c173eccfbbb169b83b Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sun, 11 Sep 2022 12:16:00 -0700 Subject: [PATCH 04/20] Describe using antlr4-tools in getting started documentation Signed-off-by: Terence Parr Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- doc/getting-started.md | 112 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/doc/getting-started.md b/doc/getting-started.md index 420556a30e..4f93acab44 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -1,10 +1,118 @@ # Getting Started with ANTLR v4 -Hi and welcome to the version 4 release of ANTLR! It's named after the fearless hero of the [Crazy Nasty-Ass Honey Badger](http://www.youtube.com/watch?v=4r7wHMg5Yjg) since ANTLR v4 takes whatever you give it--it just doesn't give a crap! See [Why do we need ANTLR v4?](faq/general.md) and the [preface of the ANTLR v4 book](http://media.pragprog.com/titles/tpantlr2/preface.pdf). +Hi and welcome to the version 4 release of ANTLR! See [Why do we need ANTLR v4?](faq/general.md) and the [preface of the ANTLR v4 book](http://media.pragprog.com/titles/tpantlr2/preface.pdf). + +## Getting started the easy way using antlr4-tools + +To play around with ANTLR without having to worry about installing it and the Java needed to execute it, use [antlr4-tools](https://github.com/antlr/antlr4-tools). The only requirement is Python3, which is typically installed on all developer machines on all operating systems. + +```bash +$ pip install antlr4-tools +``` + +That command creates `antlr4` and `antlr4-parse` executables that, if necessary, will download and install Java 11 plus the latest ANTLR jar: + +```bash +$ antlr4 +Downloading antlr4-4.11.1-complete.jar +ANTLR tool needs Java to run; install Java JRE 11 yes/no (default yes)? y +Installed Java in /Users/parrt/.jre/jdk-11.0.15+10-jre; remove that dir to uninstall +ANTLR Parser Generator Version 4.11.1 + -o ___ specify output directory where all output is generated + -lib ___ specify location of grammars, tokens files +... +``` + +On Windows, the `pip` command doesn't just work---you need to add the `...\local-packages\python38\scripts` dir to your `PATH`, which itself might require a fun reboot. If you use WSL on Windows, then the pip install will also properly at the scripts directly (if you run from bash shell). + +Let's play with a simple grammar: + +``` +grammar Expr; +prog: expr EOF ; +expr: expr ('*'|'/') expr + | expr ('+'|'-') expr + | INT + | '(' expr ')' + ; +NEWLINE : [\r\n]+ -> skip; +INT : [0-9]+ ; +``` + +### Try parsing with a sample grammar + +To parse and get the parse tree in text form, use: + +```bash +$ antlr4-parse Expr.g4 prog -tree +10+20*30 +^D +(prog:1 (expr:2 (expr:3 10) + (expr:1 (expr:3 20) * (expr:3 30))) ) +``` +(Note: `^D` means control-D and indicates "end of input" on Unix; use `^Z` on Windows.) + +Here's how to get the tokens and trace through the parse: + +```bash +$ antlr4-parse Expr.g4 prog -tokens -trace +10+20*30 +[@0,0:1='10',,1:0] +[@1,2:2='+',<'+'>,1:2] +[@2,3:4='20',,1:3] +[@3,5:5='*',<'*'>,1:5] +[@4,6:7='30',,1:6] +[@5,9:8='',,2:0] +enter prog, LT(1)=10 +enter expr, LT(1)=10 +consume [@0,0:1='10',<8>,1:0] rule expr +enter expr, LT(1)=+ +consume [@1,2:2='+',<3>,1:2] rule expr +enter expr, LT(1)=20 +consume [@2,3:4='20',<8>,1:3] rule expr +enter expr, LT(1)=* +consume [@3,5:5='*',<1>,1:5] rule expr +enter expr, LT(1)=30 +consume [@4,6:7='30',<8>,1:6] rule expr +exit expr, LT(1)= +exit expr, LT(1)= +exit expr, LT(1)= +consume [@5,9:8='',<-1>,2:0] rule prog +exit prog, LT(1)= +``` + +Here's how to get a visual tree view: + +```bash +$ antlr4-parse Expr.g4 prog -gui +10+20*30 +``` + +The following will pop up in a Java-based GUI window: + + + +### Generating parser code + +The previous section used a built-in ANTLR interpreter but typically you will ask ANTLR to generate code in the language used by your project (there are about 10 languages to choose from as of 4.11). Here's how to generate Java code from a grammar: + +```bash +$ antlr4 Expr.g4 +$ ls Expr*.java +ExprBaseListener.java ExprLexer.java ExprListener.java ExprParser.java +``` + +And, here's how to generate C++ code from the same grammar: + +```bash +$ antlr4 -Dlanguage=Cpp Expr.g4 +$ ls Expr*.cpp Expr*.h +ExprBaseListener.cpp ExprLexer.cpp ExprListener.cpp ExprParser.cpp +ExprBaseListener.h ExprLexer.h ExprListener.h ExprParser.h +``` ## Installation -ANTLR is really two things: a tool that translates your grammar to a parser/lexer in Java (or other target language) and the runtime needed by the generated parsers/lexers. Even if you are using the ANTLR Intellij plug-in or ANTLRWorks to run the ANTLR tool, the generated code will still need the runtime library. +ANTLR is really two things: a tool written in Java that translates your grammar to a parser/lexer in Java (or other target language) and the runtime library needed by the generated parsers/lexers. Even if you are using the ANTLR Intellij plug-in or ANTLRWorks to run the ANTLR tool, the generated code will still need the runtime library. The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers. In what follows, I talk about antlr-4.11.1-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3). From 7a06e1f069b75dc2a255aa24e4a0ee205b43bb4b Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sun, 11 Sep 2022 12:18:08 -0700 Subject: [PATCH 05/20] Tweak doc Signed-off-by: Terence Parr Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- doc/getting-started.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/getting-started.md b/doc/getting-started.md index 4f93acab44..8584f2aa13 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -56,6 +56,7 @@ Here's how to get the tokens and trace through the parse: ```bash $ antlr4-parse Expr.g4 prog -tokens -trace 10+20*30 +^D [@0,0:1='10',,1:0] [@1,2:2='+',<'+'>,1:2] [@2,3:4='20',,1:3] @@ -85,6 +86,7 @@ Here's how to get a visual tree view: ```bash $ antlr4-parse Expr.g4 prog -gui 10+20*30 +^D ``` The following will pop up in a Java-based GUI window: From 253ef6b7819cff80f3b4672b90bd9a0ee92811a8 Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Mon, 5 Sep 2022 00:39:59 +0200 Subject: [PATCH 06/20] Update C# release instructions Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- doc/releasing-antlr.md | 62 ++++++++---------------------------------- 1 file changed, 11 insertions(+), 51 deletions(-) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index bacfc0288b..d38c9fe692 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -279,61 +279,24 @@ zip -r ~/antlr/sites/website-antlr4/download/antlr-javascript-runtime-4.11.1.zip ### CSharp -**Install the pre-requisites** +As of writing, you can only release from a Windows box, because Visual Studio for Mac can only build the netstandard2.0 version -You need Mono and `nuget` to be installed. On mac: +**Install the pre-requisites** -- .NET build tools - can be loaded from [here](https://www.visualstudio.com/downloads/) (I need dotnet 5 and 3.1 versions) -- nuget - download [nuget.exe](https://www.nuget.org/downloads) -- dotnet - follow [the instructions here](https://www.microsoft.com/net/core) +You need 'msbuild' and `nuget` to be installed. -Alternatively, you can install Visual Studio 2017 and make sure to check boxes with .NET Core SDK. +**Creating the signed assembly** -You also need to enable .NET Framework 3.5 support in Windows "Programs and Features". +cd ~/antlr/code/antlr4/runtime/CSharp/src +dotnet build -c Release Antlr4.csproj -**Creating the signed assembly** +check that the bin/Release folder contains both the netstandard2.0 and the net45 builds +the binaries are already signed, but it's worth double checking -From Mac: +sn -v bin/Release/netstandard2.0/Antlr4.Runtime.Standard.dll +sn -v bin/Release/net45/Antlr4.Runtime.Standard.dll -```bash -critter:~ $ cd ~/antlr/code/antlr4/runtime/CSharp/src -critter:~/antlr/code/antlr4/runtime/CSharp/src $ dotnet build -c Release Antlr4.csproj -Microsoft (R) Build Engine version 16.7.2+b60ddb6f4 for .NET -Copyright (C) Microsoft Corporation. All rights reserved. - - Determining projects to restore... - Restored /Users/parrt/antlr/code/antlr4/runtime/CSharp/src/Antlr4.csproj (in 340 ms). - Antlr4 -> /Users/parrt/antlr/code/antlr4/runtime/CSharp/src/bin/Release/netstandard2.0/Antlr4.Runtime.Standard.dll - Successfully created package '/Users/parrt/antlr/code/antlr4/runtime/CSharp/src/bin/Release/Antlr4.Runtime.Standard.4.11.1.0.nupkg'. - -Build succeeded. - 0 Warning(s) - 0 Error(s) - -Time Elapsed 00:00:06.77 -critter:~/antlr/code/antlr4/runtime/CSharp/src $ /Library/Frameworks/Mono.framework/Versions/Current/Commands/sn -R bin/Release/netstandard2.0/Antlr4.Runtime.Standard.dll Antlr4.snk -Mono StrongName - version 6.12.0.0 -StrongName utility for signing assemblies -Copyright 2002, 2003 Motus Technologies. Copyright 2004-2008 Novell. BSD licensed. - -Assembly bin/Release/netstandard2.0/Antlr4.Runtime.Standard.dll signed. -critter:~/antlr/code/antlr4/runtime/CSharp/src $ /Library/Frameworks/Mono.framework/Versions/Current/Commands/sn -v bin/Release/netstandard2.0/Antlr4.Runtime.Standard.dll -Mono StrongName - version 6.12.0.0 -StrongName utility for signing assemblies -Copyright 2002, 2003 Motus Technologies. Copyright 2004-2008 Novell. BSD licensed. - -Assembly bin/Release/netstandard2.0/Antlr4.Runtime.Standard.dll is strongnamed. -$ tree /Users/parrt/antlr/code/antlr4/runtime/CSharp/src/bin/Release/ -/Users/parrt/antlr/code/antlr4/runtime/CSharp/src/bin/Release/ -├── Antlr4.Runtime.Standard.4.11.1.0.nupkg -└── netstandard2.0 - ├── Antlr4.Runtime.Standard.deps.json - ├── Antlr4.Runtime.Standard.dll - ├── Antlr4.Runtime.Standard.pdb - └── Antlr4.Runtime.Standard.xml - -1 directory, 5 files -``` +both should say the dll is valid **Publishing to NuGet** @@ -346,9 +309,6 @@ Alternately, you can publish from the cmd line. You need to get your NuGet key f nuget push Antlr4.Runtime.Standard..nupkg -Source https://www.nuget.org/api/v2/package ``` -Nuget packages are also accessible as artifacts of [AppVeyor builds](https://ci.appveyor.com/project/parrt/antlr4/build/artifacts). - - ### Python The Python targets get deployed with `setup.py` (Python 2), or `build` and `twine` (Python 3). From 871cf52c7800649ba7ce88a48e024995cfd286a1 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sun, 11 Sep 2022 15:44:04 -0700 Subject: [PATCH 07/20] Add Windows doc for antlr4-tool usage Signed-off-by: Terence Parr Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- doc/getting-started.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/getting-started.md b/doc/getting-started.md index 8584f2aa13..8831e7c097 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -4,7 +4,7 @@ Hi and welcome to the version 4 release of ANTLR! See [Why do we need ANTLR v4?] ## Getting started the easy way using antlr4-tools -To play around with ANTLR without having to worry about installing it and the Java needed to execute it, use [antlr4-tools](https://github.com/antlr/antlr4-tools). The only requirement is Python3, which is typically installed on all developer machines on all operating systems. +To play around with ANTLR without having to worry about installing it and the Java needed to execute it, use [antlr4-tools](https://github.com/antlr/antlr4-tools). The only requirement is Python3, which is typically installed on all developer machines on all operating systems. (See below for Windows issue.) ```bash $ pip install antlr4-tools @@ -23,8 +23,6 @@ ANTLR Parser Generator Version 4.11.1 ... ``` -On Windows, the `pip` command doesn't just work---you need to add the `...\local-packages\python38\scripts` dir to your `PATH`, which itself might require a fun reboot. If you use WSL on Windows, then the pip install will also properly at the scripts directly (if you run from bash shell). - Let's play with a simple grammar: ``` @@ -39,6 +37,22 @@ NEWLINE : [\r\n]+ -> skip; INT : [0-9]+ ; ``` +### Windows-specific issues + +On Windows, the `pip` command doesn't just work---you need to add the `...\local-packages\python38\scripts` dir to your `PATH`, which itself might require a fun reboot. If you use WSL on Windows, then the pip install will also properly at the scripts directly (if you run from bash shell). + + +1. Go to the Microsoft Store +2. Search in Microsoft Store for Python +3. Select the newest version of Python (3.10). +4. Click the "Get" button. Store installs python and pip at "c:\Users...\AppData\Local\Microsoft\WindowsApps\python.exe" and "c:\Users...\AppData\Local\Microsoft\WindowsApps\pip.exe", respectively. And, it updates the search path immediately with the install. +5. Open a "cmd" terminal. +6. You can now type "python" and "pip", and "pip install antlr4-tools". 7. Unfortunately, it does not add that to the search path. +7. Update the search path to contain `c:\Users...\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p8\LocalCache\local-packages\Python310\Scripts`. You may need to install MSYS2, then do a `find /c/ -name antlr4.exe 2> /dev/null` and enter that path. +8. Or, you can set up an alias to antlr4.exe on that path. + +The good news is that the ANTLR4 Python tool downloads the ANTLR jar in a standard location, and you don't need to do that manually. It's also possible to go in a browser, go to python.org, and download the python package. But, it's likely you will need to update the path for antlr4.exe as before. + ### Try parsing with a sample grammar To parse and get the parse tree in text form, use: From e20088f527f7d6bae88d81b1c8ed6abeea34434c Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Tue, 13 Sep 2022 19:31:28 -0500 Subject: [PATCH 08/20] Fixing minimum CXX standard required Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- runtime/Cpp/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/Cpp/README.md b/runtime/Cpp/README.md index 622289ba77..fb5b22da4e 100644 --- a/runtime/Cpp/README.md +++ b/runtime/Cpp/README.md @@ -35,12 +35,12 @@ The C++ target has been the work of the following people: ### Build + Usage Notes -The minimum C++ version to compile the ANTLR C++ runtime with is C++11. The supplied projects can built the runtime either as static or dynamic library, as both 32bit and 64bit arch. The macOS project contains a target for iOS and can also be built using cmake (instead of XCode). +The minimum C++ version to compile the ANTLR C++ runtime with is C++17. The supplied projects can built the runtime either as static or dynamic library, as both 32bit and 64bit arch. The macOS project contains a target for iOS and can also be built using cmake (instead of XCode). Include the antlr4-runtime.h umbrella header in your target application to get everything needed to use the library. If you are compiling with cmake, the minimum version required is cmake 2.8. -By default, the libraries produced by the CMake build target C++11. If you want to target a different C++ standard, you can explicitly pass the standard - e.g. `-DCMAKE_CXX_STANDARD=17`. +By default, the libraries produced by the CMake build target C++17. If you want to target a different C++ standard, you can explicitly pass the standard - e.g. `-DCMAKE_CXX_STANDARD=17`. #### Compiling on Windows with Visual Studio using he Visual Studio projects Simply open the VS project from the runtime folder (VS 2019+) and build it. From 6ed94f8495685b81b2251d9079326bd11714edef Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Wed, 14 Sep 2022 21:58:27 -0500 Subject: [PATCH 09/20] Fixing Reserve word NULL handling for Cpp targets Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/codegen/Target.java | 4 +++- tool/src/org/antlr/v4/codegen/model/Recognizer.java | 2 +- tool/src/org/antlr/v4/codegen/target/CppTarget.java | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 2d89281030..9d62a4c88a 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -126,7 +126,7 @@ protected void genFile(Grammar g, ST outputFileST, String fileName) * to a token type in the generated code. */ public String getTokenTypeAsTargetLabel(Grammar g, int ttype) { - String name = g.getTokenName(ttype); + String name = this.escapeIfNeeded(g.getTokenName(ttype)); // If name is not valid, return the token type instead if ( Grammar.INVALID_TOKEN_NAME.equals(name) ) { return String.valueOf(ttype); @@ -167,6 +167,7 @@ public String getTargetStringLiteralFromString(String s, boolean quoted) { return null; } + s = escapeIfNeeded(s); StringBuilder buf = new StringBuilder(); if ( quoted ) { buf.append('"'); @@ -242,6 +243,7 @@ public String getTargetStringLiteralFromANTLRStringLiteral( { StringBuilder sb = new StringBuilder(); + literal = escapeIfNeeded(literal); if ( addQuotes ) sb.append('"'); for (int i = 1; i < literal.length() -1; ) { diff --git a/tool/src/org/antlr/v4/codegen/model/Recognizer.java b/tool/src/org/antlr/v4/codegen/model/Recognizer.java index 8e07c29d2f..732f50fb20 100644 --- a/tool/src/org/antlr/v4/codegen/model/Recognizer.java +++ b/tool/src/org/antlr/v4/codegen/model/Recognizer.java @@ -59,7 +59,7 @@ public Recognizer(OutputModelFactory factory) { for (Map.Entry entry : g.tokenNameToTypeMap.entrySet()) { Integer ttype = entry.getValue(); if ( ttype>0 ) { - tokens.put(entry.getKey(), ttype); + tokens.put(gen.getTarget().escapeIfNeeded(entry.getKey()), ttype); } } diff --git a/tool/src/org/antlr/v4/codegen/target/CppTarget.java b/tool/src/org/antlr/v4/codegen/target/CppTarget.java index f8fc4d2702..b45fa91ef9 100644 --- a/tool/src/org/antlr/v4/codegen/target/CppTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CppTarget.java @@ -39,14 +39,14 @@ public class CppTarget extends Target { "double", "dynamic_cast", "else", "enum", "explicit", "export", "extern", "false", "float", "for", "friend", "goto", "if", "inline", "int", "long", "mutable", "namespace", "new", - "noexcept", "not", "not_eq", "nullptr", "operator", "or", + "noexcept", "not", "not_eq", "nullptr", "NULL", "operator", "or", "or_eq", "private", "protected", "public", "register", "reinterpret_cast", "requires", "return", "short", "signed", "sizeof", "static", "static_assert", "static_cast", "struct", "switch", "template", "this", "thread_local", "throw", "true", "try", "typedef", "typeid", "typename", "union", "unsigned", "using", "virtual", "void", "volatile", "wchar_t", "while", - "xor", "xor_eq", + "xor", "xor_eq", "rule", "parserRule" )); From 82becedeb69668745ec65a307213b0023717cf3a Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Thu, 22 Sep 2022 22:22:02 -0500 Subject: [PATCH 10/20] Adding testcase for reserve word NULL for Cpp targets Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- .../LexerExec/ReservedWordsEscaping_NULL.txt | 17 +++++++++++++++++ tool/src/org/antlr/v4/codegen/Target.java | 1 - 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt new file mode 100644 index 0000000000..0f665e1dce --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt @@ -0,0 +1,17 @@ +[notes] +https://github.com/antlr/antlr4/issues/1070 + +[type] +Lexer + +[grammar] +lexer grammar L; + +NULL : ('N' | 'n')('U' | 'u')('L' | 'l')('L' | 'l') ; + +[input] +NULL + +[output] +[@0,0:3='NULL',<1>,1:0] +[@1,4:3='',<-1>,1:4] diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 9d62a4c88a..b4b5efdea8 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -243,7 +243,6 @@ public String getTargetStringLiteralFromANTLRStringLiteral( { StringBuilder sb = new StringBuilder(); - literal = escapeIfNeeded(literal); if ( addQuotes ) sb.append('"'); for (int i = 1; i < literal.length() -1; ) { From e347ffe925a8604d17ea0718648adbc361ae1d10 Mon Sep 17 00:00:00 2001 From: "@TT" <1sand0s@users.noreply.github.com> Date: Fri, 23 Sep 2022 02:43:04 -0500 Subject: [PATCH 11/20] Update ReservedWordsEscaping_NULL.txt Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- .../descriptors/LexerExec/ReservedWordsEscaping_NULL.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt index 0f665e1dce..cd09169580 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt @@ -1,5 +1,5 @@ [notes] -https://github.com/antlr/antlr4/issues/1070 +https://github.com/antlr/antlr4/pull/3889 [type] Lexer From 57cb224225a0bbe4258eb950a441d4cc7574e338 Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Wed, 14 Sep 2022 21:58:27 -0500 Subject: [PATCH 12/20] Fixing Reserve word NULL handling for Cpp targets Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/codegen/Target.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index b4b5efdea8..9d62a4c88a 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -243,6 +243,7 @@ public String getTargetStringLiteralFromANTLRStringLiteral( { StringBuilder sb = new StringBuilder(); + literal = escapeIfNeeded(literal); if ( addQuotes ) sb.append('"'); for (int i = 1; i < literal.length() -1; ) { From 61f6aeabea91cb49f05b1408973cb3799c9c07ca Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Thu, 22 Sep 2022 22:22:02 -0500 Subject: [PATCH 13/20] Adding testcase for reserve word NULL for Cpp targets Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/codegen/Target.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 9d62a4c88a..b4b5efdea8 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -243,7 +243,6 @@ public String getTargetStringLiteralFromANTLRStringLiteral( { StringBuilder sb = new StringBuilder(); - literal = escapeIfNeeded(literal); if ( addQuotes ) sb.append('"'); for (int i = 1; i < literal.length() -1; ) { From d26c06461322a37b6eebf1f56abbbfa4745eb901 Mon Sep 17 00:00:00 2001 From: "@TT" <1sand0s@users.noreply.github.com> Date: Fri, 23 Sep 2022 16:28:29 -0500 Subject: [PATCH 14/20] Removing unecessary escape Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/codegen/Target.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index b4b5efdea8..e109850309 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -167,7 +167,6 @@ public String getTargetStringLiteralFromString(String s, boolean quoted) { return null; } - s = escapeIfNeeded(s); StringBuilder buf = new StringBuilder(); if ( quoted ) { buf.append('"'); From 62b1967c48fe98f63960b2a247703b201fac1368 Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Wed, 14 Sep 2022 21:58:27 -0500 Subject: [PATCH 15/20] Fixing Reserve word NULL handling for Cpp targets Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/codegen/Target.java | 4 +++- tool/src/org/antlr/v4/codegen/model/Recognizer.java | 2 +- tool/src/org/antlr/v4/codegen/target/CppTarget.java | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 2d89281030..9d62a4c88a 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -126,7 +126,7 @@ protected void genFile(Grammar g, ST outputFileST, String fileName) * to a token type in the generated code. */ public String getTokenTypeAsTargetLabel(Grammar g, int ttype) { - String name = g.getTokenName(ttype); + String name = this.escapeIfNeeded(g.getTokenName(ttype)); // If name is not valid, return the token type instead if ( Grammar.INVALID_TOKEN_NAME.equals(name) ) { return String.valueOf(ttype); @@ -167,6 +167,7 @@ public String getTargetStringLiteralFromString(String s, boolean quoted) { return null; } + s = escapeIfNeeded(s); StringBuilder buf = new StringBuilder(); if ( quoted ) { buf.append('"'); @@ -242,6 +243,7 @@ public String getTargetStringLiteralFromANTLRStringLiteral( { StringBuilder sb = new StringBuilder(); + literal = escapeIfNeeded(literal); if ( addQuotes ) sb.append('"'); for (int i = 1; i < literal.length() -1; ) { diff --git a/tool/src/org/antlr/v4/codegen/model/Recognizer.java b/tool/src/org/antlr/v4/codegen/model/Recognizer.java index 8e07c29d2f..732f50fb20 100644 --- a/tool/src/org/antlr/v4/codegen/model/Recognizer.java +++ b/tool/src/org/antlr/v4/codegen/model/Recognizer.java @@ -59,7 +59,7 @@ public Recognizer(OutputModelFactory factory) { for (Map.Entry entry : g.tokenNameToTypeMap.entrySet()) { Integer ttype = entry.getValue(); if ( ttype>0 ) { - tokens.put(entry.getKey(), ttype); + tokens.put(gen.getTarget().escapeIfNeeded(entry.getKey()), ttype); } } diff --git a/tool/src/org/antlr/v4/codegen/target/CppTarget.java b/tool/src/org/antlr/v4/codegen/target/CppTarget.java index f8fc4d2702..b45fa91ef9 100644 --- a/tool/src/org/antlr/v4/codegen/target/CppTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CppTarget.java @@ -39,14 +39,14 @@ public class CppTarget extends Target { "double", "dynamic_cast", "else", "enum", "explicit", "export", "extern", "false", "float", "for", "friend", "goto", "if", "inline", "int", "long", "mutable", "namespace", "new", - "noexcept", "not", "not_eq", "nullptr", "operator", "or", + "noexcept", "not", "not_eq", "nullptr", "NULL", "operator", "or", "or_eq", "private", "protected", "public", "register", "reinterpret_cast", "requires", "return", "short", "signed", "sizeof", "static", "static_assert", "static_cast", "struct", "switch", "template", "this", "thread_local", "throw", "true", "try", "typedef", "typeid", "typename", "union", "unsigned", "using", "virtual", "void", "volatile", "wchar_t", "while", - "xor", "xor_eq", + "xor", "xor_eq", "rule", "parserRule" )); From f3e3d8a75753f9044198cb622d52262d33ba8273 Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Thu, 22 Sep 2022 22:22:02 -0500 Subject: [PATCH 16/20] Adding testcase for reserve word NULL for Cpp targets Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- .../LexerExec/ReservedWordsEscaping_NULL.txt | 17 +++++++++++++++++ tool/src/org/antlr/v4/codegen/Target.java | 1 - 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt new file mode 100644 index 0000000000..0f665e1dce --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt @@ -0,0 +1,17 @@ +[notes] +https://github.com/antlr/antlr4/issues/1070 + +[type] +Lexer + +[grammar] +lexer grammar L; + +NULL : ('N' | 'n')('U' | 'u')('L' | 'l')('L' | 'l') ; + +[input] +NULL + +[output] +[@0,0:3='NULL',<1>,1:0] +[@1,4:3='',<-1>,1:4] diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 9d62a4c88a..b4b5efdea8 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -243,7 +243,6 @@ public String getTargetStringLiteralFromANTLRStringLiteral( { StringBuilder sb = new StringBuilder(); - literal = escapeIfNeeded(literal); if ( addQuotes ) sb.append('"'); for (int i = 1; i < literal.length() -1; ) { From 1eb5603f32ed755a9b1c23c77e6d0c4a8ecb531d Mon Sep 17 00:00:00 2001 From: "@TT" <1sand0s@users.noreply.github.com> Date: Fri, 23 Sep 2022 02:43:04 -0500 Subject: [PATCH 17/20] Update ReservedWordsEscaping_NULL.txt Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- .../descriptors/LexerExec/ReservedWordsEscaping_NULL.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt index 0f665e1dce..cd09169580 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/ReservedWordsEscaping_NULL.txt @@ -1,5 +1,5 @@ [notes] -https://github.com/antlr/antlr4/issues/1070 +https://github.com/antlr/antlr4/pull/3889 [type] Lexer From d08446e019c7621db47e16006947c744e85cfbe3 Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Wed, 14 Sep 2022 21:58:27 -0500 Subject: [PATCH 18/20] Fixing Reserve word NULL handling for Cpp targets Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/codegen/Target.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index b4b5efdea8..9d62a4c88a 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -243,6 +243,7 @@ public String getTargetStringLiteralFromANTLRStringLiteral( { StringBuilder sb = new StringBuilder(); + literal = escapeIfNeeded(literal); if ( addQuotes ) sb.append('"'); for (int i = 1; i < literal.length() -1; ) { From 8b60b0726993d5739f2ef1f855203344c44298f9 Mon Sep 17 00:00:00 2001 From: 1sand0s <1sand0sardpi@gmail.com> Date: Thu, 22 Sep 2022 22:22:02 -0500 Subject: [PATCH 19/20] Adding testcase for reserve word NULL for Cpp targets Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/codegen/Target.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 9d62a4c88a..b4b5efdea8 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -243,7 +243,6 @@ public String getTargetStringLiteralFromANTLRStringLiteral( { StringBuilder sb = new StringBuilder(); - literal = escapeIfNeeded(literal); if ( addQuotes ) sb.append('"'); for (int i = 1; i < literal.length() -1; ) { From bdb9d46efa52d759038dbad90b1f030d3a012a00 Mon Sep 17 00:00:00 2001 From: "@TT" <1sand0s@users.noreply.github.com> Date: Fri, 23 Sep 2022 16:28:29 -0500 Subject: [PATCH 20/20] Removing unecessary escape Signed-off-by: 1sand0s <1sand0sardpi@gmail.com> --- tool/src/org/antlr/v4/codegen/Target.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index b4b5efdea8..e109850309 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -167,7 +167,6 @@ public String getTargetStringLiteralFromString(String s, boolean quoted) { return null; } - s = escapeIfNeeded(s); StringBuilder buf = new StringBuilder(); if ( quoted ) { buf.append('"');