From 7c857692d634060062527b7f85bd0fcd038a957b Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Sun, 30 Nov 2014 16:29:35 +0800 Subject: [PATCH 1/2] add support for $self --- .../org/antlr/v4/tool/templates/codegen/Python3/Python3.stg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index c09bdf85c7..eb996ffcd0 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -575,11 +575,13 @@ RulePropertyRef_start(r) ::= "(None if . is None else . RulePropertyRef_stop(r) ::= "(None if . is None else ..stop)" RulePropertyRef_text(r) ::= "(None if . is None else self._input.getText((..start,..stop)))" RulePropertyRef_ctx(r) ::= "." +RulePropertyRef_self(r) ::= "self" ThisRulePropertyRef_start(r) ::= "localctx.start" ThisRulePropertyRef_stop(r) ::= "localctx.stop" ThisRulePropertyRef_text(r) ::= "self._input.getText((localctx.start, self._input.LT(-1)))" ThisRulePropertyRef_ctx(r) ::= "localctx" +ThisRulePropertyRef_self(r) ::= "self" NonLocalAttrRef(s) ::= "getInvokingContext()." SetNonLocalAttr(s, rhsChunks) ::= "getInvokingContext(). = " From 47d790eb1e7d6f2842f4a9acd31cce5091230c79 Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Tue, 2 Dec 2014 23:40:08 +0800 Subject: [PATCH 2/2] moved from $self to $parser also adjusted generated tests for the above and moved back to member tokens --- .../templates/codegen/Python3/Python3.stg | 39 ++++++++++--------- .../org/antlr/v4/test/rt/py3/Python3.test.stg | 12 +++++- .../antlr/v4/test/rt/py3/TestLexerExec.java | 4 +- .../antlr/v4/test/rt/py3/TestParserExec.java | 20 +++++++++- .../v4/test/rt/py3/TestSemPredEvalParser.java | 2 +- 5 files changed, 52 insertions(+), 25 deletions(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index eb996ffcd0..833f3025ac 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -110,7 +110,7 @@ class Visitor(ParseTreeVisitor): #. def visit(self, ctx): - pass + return self.visitChildren(ctx) }; separator="\n"> @@ -132,11 +132,6 @@ from . import -EOF = .EOF - -=}; separator="\n", wrap, anchor> - - class ( Parser ): grammarFileName = "" @@ -155,6 +150,11 @@ class ( Parser ): ruleNames = [ "}; separator=", ", wrap, anchor> ] + EOF = .EOF + + =}; separator="\n", wrap, anchor> + + @@ -355,7 +355,6 @@ else: >> - LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= << self.state = @@ -366,6 +365,7 @@ if : ) ) !> >> + LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << self.state = self._errHandler.sync(self) @@ -478,16 +478,16 @@ offsetShiftVar(shiftAmount, offset) ::= <% %> offsetShiftType(shiftAmount, offset) ::= <% -( - ) +(. - ). %> // produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% -==}; separator=" or "> +==.}; separator=" or "> %> cases(ttypes) ::= << -if token in [}; separator=", ">]: +if token in [.}; separator=", ">]: >> InvokeRule(r, argExprsChunks) ::= << @@ -497,7 +497,7 @@ self.state = MatchToken(m) ::= << self.state = - = }>self.match() + = }>self.match(.) >> MatchSet(m, expr, capture) ::= "" @@ -575,13 +575,13 @@ RulePropertyRef_start(r) ::= "(None if . is None else . RulePropertyRef_stop(r) ::= "(None if . is None else ..stop)" RulePropertyRef_text(r) ::= "(None if . is None else self._input.getText((..start,..stop)))" RulePropertyRef_ctx(r) ::= "." -RulePropertyRef_self(r) ::= "self" +RulePropertyRef_parser(r) ::= "self" ThisRulePropertyRef_start(r) ::= "localctx.start" ThisRulePropertyRef_stop(r) ::= "localctx.stop" ThisRulePropertyRef_text(r) ::= "self._input.getText((localctx.start, self._input.LT(-1)))" ThisRulePropertyRef_ctx(r) ::= "localctx" -ThisRulePropertyRef_self(r) ::= "self" +ThisRulePropertyRef_parser(r) ::= "self" NonLocalAttrRef(s) ::= "getInvokingContext()." SetNonLocalAttr(s, rhsChunks) ::= "getInvokingContext(). = " @@ -596,7 +596,7 @@ RuleContextListDecl(rdecl) ::= "self. = list() # of s ContextTokenGetterDecl(t) ::= << def (self): - return self.getToken(, 0) + return self.getToken(., 0) >> // should never be called @@ -608,9 +608,9 @@ def _list(self): ContextTokenListIndexedGetterDecl(t) ::= << def (self, i:int=None): if i is None: - return self.getTokens() + return self.getTokens(.) else: - return self.getToken(, i) + return self.getToken(., i) >> ContextRuleGetterDecl(r) ::= << @@ -754,6 +754,7 @@ LexerFile(lexerFile, lexer, namedActions) ::= << from antlr4 import * from io import StringIO + @@ -763,8 +764,6 @@ Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= << - = }; separator="\n", wrap, anchor> - class (Lexer): atn = ATNDeserializer().deserialize(serializedATN()) @@ -773,7 +772,9 @@ class (Lexer): = }; separator="\n"> - modeNames = [ "}; separator=", ", wrap, anchor> ] + = }; separator="\n", wrap, anchor> + + modeNames = [ "}; separator=", ", wrap, anchor> ] literalNames = [ u"\", }; separator=", ", wrap, anchor> ] diff --git a/tool/test/org/antlr/v4/test/rt/py3/Python3.test.stg b/tool/test/org/antlr/v4/test/rt/py3/Python3.test.stg index 5668b24b31..03777fa382 100644 --- a/tool/test/org/antlr/v4/test/rt/py3/Python3.test.stg +++ b/tool/test/org/antlr/v4/test/rt/py3/Python3.test.stg @@ -179,6 +179,14 @@ RuleInvocationStack() ::= "str_list(self.getRuleInvocationStack())" LL_EXACT_AMBIG_DETECTION() ::= <> +ParserPropertyMember() ::= << +@members { +def Property(self): + return True + +} +>> + PositionAdjustingLexer() ::= << def resetAcceptPosition(self, index, line, column): @@ -193,9 +201,9 @@ def nextToken(self): return super(type(self),self).nextToken() def emit(self): - if self._type==TOKENS: + if self._type==PositionAdjustingLexer.TOKENS: self.handleAcceptPositionForKeyword("tokens") - elif self._type==LABEL: + elif self._type==PositionAdjustingLexer.LABEL: self.handleAcceptPositionForIdentifier() return super(type(self),self).emit() diff --git a/tool/test/org/antlr/v4/test/rt/py3/TestLexerExec.java b/tool/test/org/antlr/v4/test/rt/py3/TestLexerExec.java index b1b58c7b18..55bca93093 100644 --- a/tool/test/org/antlr/v4/test/rt/py3/TestLexerExec.java +++ b/tool/test/org/antlr/v4/test/rt/py3/TestLexerExec.java @@ -608,9 +608,9 @@ public void testPositionAdjustingLexer() throws Exception { sb.append(" return super(type(self),self).nextToken()\n"); sb.append("\n"); sb.append("def emit(self):\n"); - sb.append(" if self._type==TOKENS:\n"); + sb.append(" if self._type==PositionAdjustingLexer.TOKENS:\n"); sb.append(" self.handleAcceptPositionForKeyword(\"tokens\")\n"); - sb.append(" elif self._type==LABEL:\n"); + sb.append(" elif self._type==PositionAdjustingLexer.LABEL:\n"); sb.append(" self.handleAcceptPositionForIdentifier()\n"); sb.append(" return super(type(self),self).emit()\n"); sb.append("\n"); diff --git a/tool/test/org/antlr/v4/test/rt/py3/TestParserExec.java b/tool/test/org/antlr/v4/test/rt/py3/TestParserExec.java index 451b89be0e..ae23caa412 100644 --- a/tool/test/org/antlr/v4/test/rt/py3/TestParserExec.java +++ b/tool/test/org/antlr/v4/test/rt/py3/TestParserExec.java @@ -332,7 +332,7 @@ public void testPredicatedIfIfElse() throws Exception { String grammar = "grammar T;\n" + "s : stmt EOF ;\n" + "stmt : ifStmt | ID;\n" + - "ifStmt : 'if' ID stmt ('else' stmt | { self._input.LA(1)!=ELSE }?);\n" + + "ifStmt : 'if' ID stmt ('else' stmt | { self._input.LA(1)!=TParser.ELSE }?);\n" + "ELSE : 'else';\n" + "ID : [a-zA-Z]+;\n" + "WS : [ \\n\\t]+ -> skip;"; @@ -453,5 +453,23 @@ public void testReferenceToATN_2() throws Exception { assertNull(this.stderrDuringParse); } + /* this file and method are generated, any edit will be overwritten by the next generation */ + @Test + public void testParserProperty() throws Exception { + String grammar = "grammar T;\n" + + "@members {\n" + + "def Property(self):\n" + + " return True\n" + + "\n" + + "}\n" + + "a : {$parser.Property()}? ID {print(\"valid\")}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "WS : (' '|'\\n') -> skip ;"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "a", "abc", false); + assertEquals("valid\n", found); + assertNull(this.stderrDuringParse); + } + } \ No newline at end of file diff --git a/tool/test/org/antlr/v4/test/rt/py3/TestSemPredEvalParser.java b/tool/test/org/antlr/v4/test/rt/py3/TestSemPredEvalParser.java index e318696b43..f822cf8f9a 100644 --- a/tool/test/org/antlr/v4/test/rt/py3/TestSemPredEvalParser.java +++ b/tool/test/org/antlr/v4/test/rt/py3/TestSemPredEvalParser.java @@ -427,7 +427,7 @@ String testPredFromAltTestedInLoopBack(String input) throws Exception { "@after {print($ctx.toStringTree(recog=self))}\n" + " : para para EOF ;\n" + "para: paraContent NL NL ;\n" + - "paraContent : ('s'|'x'|{self._input.LA(2)!=NL}? NL)+ ;\n" + + "paraContent : ('s'|'x'|{self._input.LA(2)!=TParser.NL}? NL)+ ;\n" + "NL : '\\n' ;\n" + "s : 's' ;\n" + "X : 'x' ;";