[antlr4 grammar/CI build] Fix for 2609

.github/workflows/main.yml

@@ -127,8 +127,8 @@ jobs:
     - name: Install trgen
       shell: pwsh
       run: |
-        dotnet tool install -g trgen --version 0.16.1
-        dotnet tool install -g trwdog --version 0.16.1
+        dotnet tool install -g trgen --version 0.16.3
+        dotnet tool install -g trwdog --version 0.16.3
         if ("${{ matrix.os }}" -eq "ubuntu-latest") {
             echo "$HOME/.dotnet/tools" >> $env:GITHUB_PATH
         }

_scripts/really-run.ps1

@@ -14,8 +14,8 @@ $antlrPath = _scripts/get-antlr.ps1 "4.10"
 # Set up env as it is used in test script.
 echo "antlr_path=$antlrPath" >> $env:GITHUB_ENV
 
-dotnet tool install -g trgen --version 0.16.1
-dotnet tool install -g trwdog --version 0.16.1
+dotnet tool install -g trgen --version 0.16.3
+dotnet tool install -g trwdog --version 0.16.3
 
 # Call test script.
 $env:ANTLR_JAR_PATH="$antlrPath"

_scripts/regtest.sh

@@ -164,11 +164,11 @@ setupdeps()
     date
     echo "Setting up trgen and antlr jar."
     dotnet tool uninstall -g trgen
-    dotnet tool install -g trgen --version 0.16.1
+    dotnet tool install -g trgen --version 0.16.3
     dotnet tool uninstall -g trxml2
-    dotnet tool install -g trxml2 --version 0.16.1
+    dotnet tool install -g trxml2 --version 0.16.3
     dotnet tool uninstall -g trwdog
-    dotnet tool install -g trwdog --version 0.16.1
+    dotnet tool install -g trwdog --version 0.16.3
 	case "${unameOut}" in
 		Linux*)     curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.10/antlr4-4.10-complete.jar' -o /tmp/antlr4-4.10-complete.jar;;
 		Darwin*)    curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.10/antlr4-4.10-complete.jar' -o /tmp/antlr4-4.10-complete.jar;;

_scripts/skip-cpp.txt

@@ -6,7 +6,6 @@ alpaca
 angelscript
 antlr/antlr2
 antlr/antlr3
-antlr/antlr4
 apex
 apt
 arithmetic

_scripts/skip-python3.txt

@@ -1,7 +1,6 @@
 _grammar-test
 antlr/antlr2
 antlr/antlr3
-antlr/antlr4
 apex
 asm/asmMASM
 asm/masm

antlr/antlr4/ANTLRv4Lexer.g4

@@ -328,6 +328,11 @@ ARGUMENT_CONTENT
    : .
    ;
 
+// TODO: This grammar and the one used in the Intellij Antlr4 plugin differ
+// for "actions". This needs to be resolved at some point.
+// The Intellij Antlr4 grammar is here:
+// https://github.com/antlr/intellij-plugin-v4/blob/1f36fde17f7fa63cb18d7eeb9cb213815ac658fb/src/main/antlr/org/antlr/intellij/plugin/parser/ANTLRv4Lexer.g4#L587
+
 // -------------------------
 // Target Language Actions
 //

antlr/antlr4/ANTLRv4Parser.g4

@@ -212,7 +212,7 @@ labeledAlt
    // Lexer rules
 
 lexerRuleSpec
-   : FRAGMENT? TOKEN_REF COLON lexerRuleBlock SEMI
+   : FRAGMENT? TOKEN_REF optionsSpec? COLON lexerRuleBlock SEMI
    ;
 
 lexerRuleBlock

antlr/antlr4/CSharp/LexerAdaptor.cs

@@ -35,6 +35,7 @@ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 public abstract class LexerAdaptor : Lexer
 {
     private static readonly int PREQUEL_CONSTRUCT = -10;
+    private static readonly int OPTIONS_CONSTRUCT = -11;
 
     // I copy a reference to the stream, so It can be used as a Char Stream, not as a IISStream
     readonly ICharStream stream;
@@ -45,16 +46,16 @@ public abstract class LexerAdaptor : Lexer
     protected LexerAdaptor(ICharStream input)
          : base(input, Console.Out, Console.Error)
     {
-	    CurrentRuleType = TokenConstants.InvalidType;
-	    _insideOptionsBlock = false;
+        CurrentRuleType = TokenConstants.InvalidType;
+        _insideOptionsBlock = false;
         stream = input;
     }
 
     protected LexerAdaptor(ICharStream input, TextWriter output, TextWriter errorOutput)
          : base(input, output, errorOutput)
     {
-	    CurrentRuleType = TokenConstants.InvalidType;
-	    _insideOptionsBlock = false;
+        CurrentRuleType = TokenConstants.InvalidType;
+        _insideOptionsBlock = false;
         stream = input;
     }
 
@@ -107,20 +108,6 @@ protected void handleEndAction()
         }
     }
 
-    protected void handleOptionsLBrace()
-    {
-        if (_insideOptionsBlock)
-        {
-            Type = ANTLRv4Lexer.BEGIN_ACTION;
-            PushMode(ANTLRv4Lexer.TargetLanguageAction);
-        }
-        else
-        {
-            Type = ANTLRv4Lexer.LBRACE;
-            _insideOptionsBlock = true;
-        }
-    }
-
     private bool InLexerRule
     {
         get { return CurrentRuleType == ANTLRv4Lexer.TOKEN_REF; }
@@ -133,16 +120,27 @@ public override IToken Emit()
             // enter prequel construct ending with an RBRACE
             CurrentRuleType = PREQUEL_CONSTRUCT;
         }
+        else if (Type == ANTLRv4Lexer.OPTIONS && CurrentRuleType == ANTLRv4Lexer.TOKEN_REF)
+        {
+            CurrentRuleType = OPTIONS_CONSTRUCT;
+        }
         else if (Type == ANTLRv4Lexer.RBRACE && CurrentRuleType == PREQUEL_CONSTRUCT)
         {
             // exit prequel construct
             CurrentRuleType = TokenConstants.InvalidType;
         }
+        else if (Type == ANTLRv4Lexer.RBRACE && CurrentRuleType == OPTIONS_CONSTRUCT)
+        { // exit options
+            CurrentRuleType = ANTLRv4Lexer.TOKEN_REF;
+        }
         else if (Type == ANTLRv4Lexer.AT && CurrentRuleType == TokenConstants.InvalidType)
         {
             // enter action
             CurrentRuleType = ANTLRv4Lexer.AT;
         }
+        else if (Type == ANTLRv4Lexer.SEMI && CurrentRuleType == OPTIONS_CONSTRUCT)
+        { // ';' in options { .... }. Don't change anything.
+        }
         else if (Type == ANTLRv4Lexer.END_ACTION && CurrentRuleType == ANTLRv4Lexer.AT)
         {
             // exit action
@@ -155,7 +153,6 @@ public override IToken Emit()
             {
                 Type = ANTLRv4Lexer.TOKEN_REF;
             }
-
             if (char.IsLower(firstChar))
             {
                 Type = ANTLRv4Lexer.RULE_REF;
@@ -177,8 +174,8 @@ public override IToken Emit()
 
     public override void Reset()
     {
-	    CurrentRuleType = TokenConstants.InvalidType;
-	    _insideOptionsBlock = false;
+        CurrentRuleType = TokenConstants.InvalidType;
+        _insideOptionsBlock = false;
         base.Reset();
     }
 }