Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Various cleanup, plus fixes the way a predicate is determined to be in or out of context #15

Merged
merged 13 commits into from
Feb 20, 2012
Merged

Various cleanup, plus fixes the way a predicate is determined to be in or out of context #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6304a56
Fix unit tests
sharwell Feb 9, 2012
b686efb
Code cleanup in unit tests
sharwell Feb 9, 2012
bf5df30
Remove unnecessary initializations, use isEmpty() instead of size()==0
sharwell Feb 9, 2012
d589011
Small fixes to the Java grammars used in the performance test
sharwell Feb 9, 2012
7fb73a3
Construct DFA instances with the decision number
sharwell Feb 9, 2012
702b91f
Ignore TestPerformance by default since it fails without special conf…
sharwell Feb 9, 2012
ddf946b
Token.EOF instead of -1, add <?> to generic type usage, make getUniqu…
sharwell Feb 9, 2012
4677977
Fix generics in ParserATNSimulator
sharwell Feb 9, 2012
99ce3cb
Fix ParserATNSimulator.closure not tracking proper context for contex…
sharwell Feb 10, 2012
30c58bf
Specify <?> type parameter on raw types
sharwell Feb 10, 2012
cd3adb1
Fix ParserATNSimulator.execDFA adding incorrect error edges in the DFA
sharwell Feb 10, 2012
9e192fe
Merge remote-tracking branch 'parrt/master'
sharwell Feb 10, 2012
27806dc
Merge remote-tracking branch 'parrt/master'
sharwell Feb 15, 2012
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 70 additions & 61 deletions runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ So, we now cover what I hope is the vast majority of the cases (in
* when closure operations fall off the end of the rule that
* holds the decision were evaluating
*/
public class ParserATNSimulator<Symbol> extends ATNSimulator {
public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
public static boolean debug = false;
public static boolean dfa_debug = false;
public static boolean retry_debug = false;
Expand Down Expand Up @@ -261,15 +261,14 @@ public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn) {
public void reset() {
}

public int adaptivePredict(@NotNull SymbolStream<Token> input, int decision,
@Nullable ParserRuleContext outerContext)
public int adaptivePredict(@NotNull SymbolStream<? extends Symbol> input, int decision,
@Nullable ParserRuleContext<?> outerContext)
{
predict_calls++;
DFA dfa = decisionToDFA[decision];
if ( dfa==null || dfa.s0==null ) {
DecisionState startState = atn.decisionToState.get(decision);
decisionToDFA[decision] = dfa = new DFA(startState);
dfa.decision = decision;
decisionToDFA[decision] = dfa = new DFA(startState, decision);
return predictATN(dfa, input, outerContext);
}
else {
Expand All @@ -288,8 +287,8 @@ public int adaptivePredict(@NotNull SymbolStream<Token> input, int decision,
}
}

public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream<Token> input,
@Nullable ParserRuleContext outerContext)
public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream<? extends Symbol> input,
@Nullable ParserRuleContext<?> outerContext)
{
if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
if ( debug ) System.out.println("ATN decision "+dfa.decision+
Expand Down Expand Up @@ -320,8 +319,8 @@ public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream<Token> input,
}

public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0,
@NotNull SymbolStream<Token> input, int startIndex,
@Nullable ParserRuleContext outerContext)
@NotNull SymbolStream<? extends Symbol> input, int startIndex,
@Nullable ParserRuleContext<?> outerContext)
{
if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
Expand Down Expand Up @@ -371,37 +370,32 @@ public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0,
// if no edge, pop over to ATN interpreter, update DFA and return
if ( s.edges == null || t >= s.edges.length || t < -1 || s.edges[t+1] == null ) {
if ( dfa_debug && t>=0 ) System.out.println("no edge for "+parser.getTokenNames()[t]);
int alt = -1;
int alt;
if ( dfa_debug ) {
System.out.println("ATN exec upon "+
parser.getInputString(startIndex) +
" at DFA state "+s.stateNumber);
}
try {
alt = execATN(dfa, s, input, startIndex, outerContext);
// this adds edge even if next state is accept for
// same alt; e.g., s0-A->:s1=>2-B->:s2=>2
// TODO: This next stuff kills edge, but extra states remain. :(
if ( s.isAcceptState && alt!=-1 ) {
DFAState d = s.edges[input.LA(1)+1];
if ( d.isAcceptState && d.prediction==s.prediction ) {
// we can carve it out.
s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error
}
}
if ( dfa_debug ) {
System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames()));
//dump(dfa);

alt = execATN(dfa, s, input, startIndex, outerContext);
// this adds edge even if next state is accept for
// same alt; e.g., s0-A->:s1=>2-B->:s2=>2
// TODO: This next stuff kills edge, but extra states remain. :(
if ( s.isAcceptState && alt!=-1 ) {
DFAState d = s.edges[input.LA(1)+1];
if ( d.isAcceptState && d.prediction==s.prediction ) {
// we can carve it out.
s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error
}
// action already executed
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" predicts "+alt);
return alt; // we've updated DFA, exec'd action, and have our deepest answer
}
catch (NoViableAltException nvae) {
addDFAEdge(s, t, ERROR);
throw nvae;
if ( dfa_debug ) {
System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames()));
//dump(dfa);
}
// action already executed
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" predicts "+alt);
return alt; // we've updated DFA, exec'd action, and have our deepest answer
}
DFAState target = s.edges[t+1];
if ( target == ERROR ) {
Expand Down Expand Up @@ -479,15 +473,15 @@ cases except no viable alternative (and possibly non-greedy decisions?)

*/
public int execATN(@NotNull DFA dfa, @NotNull DFAState s0,
@NotNull SymbolStream<Token> input, int startIndex,
ParserRuleContext outerContext)
@NotNull SymbolStream<? extends Symbol> input, int startIndex,
ParserRuleContext<?> outerContext)
{
if ( debug ) System.out.println("execATN decision "+dfa.decision+" exec LA(1)=="+ getLookaheadName(input));
ATN_failover++;

ATNConfigSet previous = s0.configset;
DFAState D = null;
ATNConfigSet fullCtxSet = null;
DFAState D;
ATNConfigSet fullCtxSet;

if ( debug ) System.out.println("s0 = "+s0);

Expand Down Expand Up @@ -600,8 +594,8 @@ public int execATN(@NotNull DFA dfa, @NotNull DFAState s0,
public ATNConfigSet execATNWithFullContext(DFA dfa,
DFAState D, // how far we got before failing over
@NotNull ATNConfigSet s0,
@NotNull SymbolStream<Token> input, int startIndex,
ParserRuleContext outerContext,
@NotNull SymbolStream<? extends Symbol> input, int startIndex,
ParserRuleContext<?> outerContext,
int nalts,
boolean greedy)
{
Expand Down Expand Up @@ -637,7 +631,7 @@ public ATNConfigSet execATNWithFullContext(DFA dfa,
if ( reach.hasSemanticContext ) {
SemanticContext[] altToPred = getPredsForAmbigAlts(reach.conflictingAlts, reach, nalts);
// altToPred[uniqueAlt] is now our validating predicate (if any)
List<DFAState.PredPrediction> predPredictions = null;
List<DFAState.PredPrediction> predPredictions;
if ( altToPred!=null ) {
// we have a validating predicate; test it
predPredictions = getPredicatePredictions(reach.conflictingAlts, altToPred);
Expand Down Expand Up @@ -799,7 +793,7 @@ public List<DFAState.PredPrediction> getPredicatePredictions(IntervalSet ambigAl
pairs.add(new DFAState.PredPrediction(pred, i));
}
}
if ( pairs.size()==0 ) pairs = null;
if ( pairs.isEmpty() ) pairs = null;
else if ( firstUnpredicated!=ATN.INVALID_ALT_NUMBER ) {
// add default prediction if we found null predicate
pairs.add(new DFAState.PredPrediction(null, firstUnpredicated));
Expand All @@ -813,7 +807,7 @@ else if ( firstUnpredicated!=ATN.INVALID_ALT_NUMBER ) {
* prediction for disambiguating predicates.
*/
public int evalSemanticContext(List<DFAState.PredPrediction> predPredictions,
ParserRuleContext outerContext)
ParserRuleContext<?> outerContext)
{
int predictedAlt = ATN.INVALID_ALT_NUMBER;
// List<DFAState.PredPrediction> predPredictions = D.predicates;
Expand Down Expand Up @@ -851,6 +845,17 @@ protected void closure(@NotNull ATNConfig config,
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean greedy, boolean loopsSimulateTailRecursion)
{
final int initialDepth = 0;
closure(config, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, initialDepth);
}

protected void closure(@NotNull ATNConfig config,
@NotNull ATNConfigSet configs,
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean greedy, boolean loopsSimulateTailRecursion,
int depth)
{
if ( debug ) System.out.println("closure("+config.toString(parser,true)+")");

Expand All @@ -875,7 +880,7 @@ protected void closure(@NotNull ATNConfig config,
// gotten that context AFTER having falling off a rule.
// Make sure we track that we are now out of context.
c.reachesIntoOuterContext = config.reachesIntoOuterContext;
closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion);
closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, depth - 1);
return;
}
else {
Expand Down Expand Up @@ -919,8 +924,9 @@ else if ( config.state.getClass()==LoopEndState.class ) {
Transition t = p.transition(i);
boolean continueCollecting =
!(t instanceof ActionTransition) && collectPredicates;
ATNConfig c = getEpsilonTarget(config, t, continueCollecting);
ATNConfig c = getEpsilonTarget(config, t, continueCollecting, depth == 0);
if ( c!=null ) {
int newDepth = depth;
if ( config.state instanceof RuleStopState ) {
// target fell off end of rule; mark resulting c as having dipped into outer context
// We can't get here if incoming config was rule stop and we had context
Expand All @@ -929,9 +935,16 @@ else if ( config.state.getClass()==LoopEndState.class ) {
// preds if this is > 0.
c.reachesIntoOuterContext++;
configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method
newDepth--;
if ( debug ) System.out.println("dips into outer ctx: "+c);
}
closure(c, configs, closureBusy, continueCollecting, greedy, loopsSimulateTailRecursion);
else if (t instanceof RuleTransition) {
if (newDepth >= 0) {
newDepth++;
}
}

closure(c, configs, closureBusy, continueCollecting, greedy, loopsSimulateTailRecursion, newDepth);
}
}
}
Expand All @@ -943,12 +956,12 @@ public String getRuleName(int index) {
}

@Nullable
public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates) {
public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates, boolean inContext) {
if ( t instanceof RuleTransition ) {
return ruleTransition(config, t);
}
else if ( t instanceof PredicateTransition ) {
return predTransition(config, (PredicateTransition)t, collectPredicates);
return predTransition(config, (PredicateTransition)t, collectPredicates, inContext);
}
else if ( t instanceof ActionTransition ) {
return actionTransition(config, (ActionTransition)t);
Expand All @@ -968,7 +981,8 @@ public ATNConfig actionTransition(@NotNull ATNConfig config, @NotNull ActionTran
@Nullable
public ATNConfig predTransition(@NotNull ATNConfig config,
@NotNull PredicateTransition pt,
boolean collectPredicates)
boolean collectPredicates,
boolean inContext)
{
if ( debug ) {
System.out.println("PRED (collectPredicates="+collectPredicates+") "+
Expand All @@ -979,12 +993,6 @@ public ATNConfig predTransition(@NotNull ATNConfig config,
parser.getRuleInvocationStack());
}
}
// We know the correct context in exactly one spot: in the original
// rule that invokes the ATN simulation. We know we are in this rule
// when the context stack is empty and we've not dipped into
// the outer context.
boolean inContext =
config.context==ParserRuleContext.EMPTY && config.reachesIntoOuterContext==0;

ATNConfig c;
if ( collectPredicates &&
Expand Down Expand Up @@ -1207,7 +1215,7 @@ protected int resolveNongreedyToExitBranch(@NotNull ATNConfigSet reach,

@NotNull
public String getTokenName(int t) {
if ( t==-1 ) return "EOF";
if ( t==Token.EOF ) return "EOF";
if ( parser!=null && parser.getTokenNames()!=null ) {
String[] tokensNames = parser.getTokenNames();
if ( t>=tokensNames.length ) {
Expand All @@ -1221,7 +1229,7 @@ public String getTokenName(int t) {
return String.valueOf(t);
}

public String getLookaheadName(SymbolStream<Token> input) {
public String getLookaheadName(SymbolStream<? extends Symbol> input) {
return getTokenName(input.LA(1));
}

Expand All @@ -1246,18 +1254,18 @@ else if ( t instanceof SetTransition ) {
}

@NotNull
public NoViableAltException noViableAlt(@NotNull SymbolStream<Token> input,
@NotNull ParserRuleContext outerContext,
public NoViableAltException noViableAlt(@NotNull SymbolStream<? extends Symbol> input,
@NotNull ParserRuleContext<?> outerContext,
@NotNull ATNConfigSet configs,
int startIndex)
{
return new NoViableAltException(parser, input,
(Token)input.get(startIndex),
(Token)input.LT(1),
input.get(startIndex),
input.LT(1),
configs, outerContext);
}

public static int getUniqueAlt(@NotNull Collection<ATNConfig> configs) {
public int getUniqueAlt(@NotNull Collection<ATNConfig> configs) {
int alt = ATN.INVALID_ALT_NUMBER;
for (ATNConfig c : configs) {
if ( alt == ATN.INVALID_ALT_NUMBER ) {
Expand All @@ -1282,6 +1290,7 @@ public boolean configWithAltAtStopState(@NotNull Collection<ATNConfig> configs,
return false;
}

@NotNull
protected DFAState addDFAEdge(@NotNull DFA dfa,
@NotNull ATNConfigSet p,
int t,
Expand All @@ -1304,7 +1313,7 @@ protected void addDFAEdge(@Nullable DFAState p, int t, @Nullable DFAState q) {
}

/** See comment on LexerInterpreter.addDFAState. */
@Nullable
@NotNull
protected DFAState addDFAState(@NotNull DFA dfa, @NotNull ATNConfigSet configs) {
DFAState proposed = new DFAState(configs);
DFAState existing = dfa.states.get(proposed);
Expand Down
12 changes: 10 additions & 2 deletions runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ public class DFA {
public final Map<DFAState, DFAState> states = new LinkedHashMap<DFAState, DFAState>();
@Nullable
public DFAState s0;
public int decision;

public final int decision;

/** From which ATN state did we create this DFA? */
@NotNull
Expand All @@ -54,7 +55,14 @@ public class DFA {
*/
// public OrderedHashSet<ATNConfig> conflictSet;

public DFA(@NotNull DecisionState atnStartState) { this.atnStartState = atnStartState; }
public DFA(@NotNull DecisionState atnStartState) {
this(atnStartState, 0);
}

public DFA(@NotNull DecisionState atnStartState, int decision) {
this.atnStartState = atnStartState;
this.decision = decision;
}

/** Find the path in DFA from s0 to s, returning list of states encountered (inclusively) */
// public List<DFAState> getPathToState(DFAState finalState, TokenStream input, int start, int stop) {
Expand Down
2 changes: 2 additions & 0 deletions tool/test/org/antlr/v4/test/Java.g
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,9 @@ compilationUnit
( packageDeclaration importDeclaration* typeDeclaration*
| classOrInterfaceDeclaration typeDeclaration*
)
EOF
| packageDeclaration? importDeclaration* typeDeclaration*
EOF
;

packageDeclaration
Expand Down
8 changes: 4 additions & 4 deletions tool/test/org/antlr/v4/test/TestATNParserPrediction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.runtime.NoViableAltException;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.DFA;
Expand Down Expand Up @@ -504,8 +505,7 @@ public void checkPredictedAlt(LexerGrammar lg, Grammar g, int decision,
TokenStream input = new IntTokenStream(types);
ParserInterpreter interp = new ParserInterpreter(g, input);
DecisionState startState = atn.decisionToState.get(decision);
DFA dfa = new DFA(startState);
dfa.decision = decision;
DFA dfa = new DFA(startState, decision);
int alt = interp.predictATN(dfa, input, ParserRuleContext.EMPTY, false);

System.out.println(dot.getDOT(dfa, false));
Expand All @@ -523,7 +523,7 @@ public void checkPredictedAlt(LexerGrammar lg, Grammar g, int decision,
}

public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName,
String inputString, ParserRuleContext ctx)
String inputString, ParserRuleContext<?> ctx)
{
Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg);
Expand All @@ -541,7 +541,7 @@ public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName,
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b"))));
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e"))));

ParserATNSimulator interp = new ParserATNSimulator(atn);
ParserATNSimulator<Token> interp = new ParserATNSimulator<Token>(atn);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);
TokenStream input = new IntTokenStream(types);
Expand Down
Loading