Big update to the listener events

.gitignore

@@ -0,0 +1,3 @@
+/tool/target/
+/runtime/Java/target/
+/gunit/target/

CHANGES.txt

@@ -1,5 +1,9 @@
 ANTLR v4 Honey Badger early access
 
+Feb 14, 2012
+
+* Fixed https://github.com/antlr/antlr4/issues/8 and lots of other little things.
+
 Jan 30, 2012
 
 * Moving to github.

gunit/pom.xml

@@ -0,0 +1,80 @@
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.antlr</groupId>
+    <artifactId>antlr4-gunit</artifactId>
+    <version>4.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>antlr4-gunit</name>
+    <url>http://www.antlr.org</url>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.10</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.antlr</groupId>
+            <artifactId>antlr4-runtime</artifactId>
+            <version>4.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.antlr</groupId>
+            <artifactId>antlr-runtime</artifactId>
+            <version>3.4.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.antlr</groupId>
+            <artifactId>ST4</artifactId>
+            <version>4.0.4</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+
+        <sourceDirectory>src</sourceDirectory>
+        <resources>
+            <resource>
+                <directory>resources</directory>
+            </resource>
+        </resources>
+
+        <plugins>
+           <plugin>
+                <groupId>org.antlr</groupId>
+                <artifactId>antlr3-maven-plugin</artifactId>
+                <version>3.4</version>
+                <configuration>
+                    <sourceDirectory>src</sourceDirectory>
+                    <verbose>true</verbose>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>antlr</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.3.2</version>
+                <configuration>
+                    <source>1.6</source>
+                    <target>1.6</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

runtime/Java/pom.xml

@@ -0,0 +1,50 @@
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.antlr</groupId>
+    <artifactId>antlr4-runtime</artifactId>
+    <version>4.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>antlr4-runtime</name>
+    <url>http://www.antlr.org</url>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.antlr</groupId>
+            <artifactId>ST4</artifactId>
+            <version>4.0.4-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.abego</groupId>
+            <artifactId>treelayout.core</artifactId>
+            <version>1.0</version>
+            <scope>system</scope>
+            <systemPath>${project.basedir}/lib/org.abego.treelayout.core.jar</systemPath>
+        </dependency>
+    </dependencies>
+
+    <build>
+
+        <sourceDirectory>src</sourceDirectory>
+
+        <plugins>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.3.2</version>
+                <configuration>
+                    <source>1.6</source>
+                    <target>1.6</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java

@@ -1,6 +1,7 @@
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.atn.ATNConfig;
+import org.antlr.v4.runtime.atn.ATNConfigSet;
 import org.antlr.v4.runtime.atn.DecisionState;
 import org.antlr.v4.runtime.atn.SemanticContext;
 import org.antlr.v4.runtime.dfa.DFA;
@@ -125,12 +126,12 @@ void reportError(@NotNull Parser recognizer,
      */
     void reportAmbiguity(@NotNull Parser recognizer,
 						 DFA dfa, int startIndex, int stopIndex, @NotNull IntervalSet ambigAlts,
-						 @NotNull OrderedHashSet<ATNConfig> configs);
+						 @NotNull ATNConfigSet configs);
 
 	void reportAttemptingFullContext(@NotNull Parser recognizer,
 									 @NotNull DFA dfa,
 									 int startIndex, int stopIndex,
-									 @NotNull OrderedHashSet<ATNConfig> configs);
+									 @NotNull ATNConfigSet configs);
 
 	/** Called by the parser when it find a conflict that is resolved by retrying the parse
      *  with full context. This is not a warning; it simply notifies you that your grammar
@@ -140,7 +141,7 @@ void reportAttemptingFullContext(@NotNull Parser recognizer,
     void reportContextSensitivity(@NotNull Parser recognizer,
                                   @NotNull DFA dfa,
                                   int startIndex, int stopIndex,
-                                  @NotNull OrderedHashSet<ATNConfig> configs);
+                                  @NotNull ATNConfigSet configs);
 
     /** Called by the parser when it finds less than n-1 predicates for n ambiguous alternatives.
      *  If there are n-1, we assume that the missing predicate is !(the "or" of the other predicates).
@@ -152,5 +153,5 @@ void reportInsufficientPredicates(@NotNull Parser recognizer,
 									  int startIndex, int stopIndex, @NotNull IntervalSet ambigAlts,
 									  DecisionState decState,
 									  @NotNull SemanticContext[] altToPred,
-									  @NotNull OrderedHashSet<ATNConfig> configs, boolean fullContextParse);
+									  @NotNull ATNConfigSet configs, boolean fullContextParse);
 }

runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java

@@ -81,24 +81,14 @@ public BufferedTokenStream(TokenSource tokenSource) {
 
     @Override
     public int mark() {
-        if ( p == -1 ) setup();
-		lastMarker = index();
-		return lastMarker;
+		return 0;
 	}
 
 	@Override
 	public void release(int marker) {
 		// no resources to release
 	}
 
-    public void rewind(int marker) {
-        seek(marker);
-    }
-
-    public void rewind() {
-        seek(lastMarker);
-    }
-
     public void reset() {
         p = 0;
         lastMarker = 0;

runtime/Java/src/org/antlr/v4/runtime/CommonToken.java

@@ -64,8 +64,10 @@ public CommonToken(TokenSource source, int type, int channel, int start, int sto
 		this.channel = channel;
 		this.start = start;
 		this.stop = stop;
-        this.line = source.getLine();
-        this.charPositionInLine = source.getCharPositionInLine();
+		if (source != null) {
+			this.line = source.getLine();
+			this.charPositionInLine = source.getCharPositionInLine();
+		}
 	}
 
 	public CommonToken(int type, String text) {
@@ -190,7 +192,7 @@ public TokenSource getTokenSource() {
 	}
 
 	public CharStream getInputStream() {
-		return source.getInputStream();
+		return source != null ? source.getInputStream() : null;
 	}
 
 	public String toString() {

runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java

@@ -83,6 +83,12 @@ public void seek(int index) {
         }
     }
 
+	@Override
+	public void reset() {
+		super.reset();
+		p = skipOffTokenChannels(p);
+	}
+
     @Override
     protected CommonToken LB(int k) {
         if ( k==0 || (p-k)<0 ) return null;

runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java

@@ -554,21 +554,21 @@ public void consumeUntil(Parser recognizer, IntervalSet set) {
     @Override
     public void reportAmbiguity(@NotNull Parser recognizer,
 								DFA dfa, int startIndex, int stopIndex, @NotNull IntervalSet ambigAlts,
-								@NotNull OrderedHashSet<ATNConfig> configs)
+								@NotNull ATNConfigSet configs)
     {
     }
 
 	@Override
 	public void reportAttemptingFullContext(@NotNull Parser recognizer,
 											@NotNull DFA dfa,
 											int startIndex, int stopIndex,
-											@NotNull OrderedHashSet<ATNConfig> configs)
+											@NotNull ATNConfigSet configs)
 	{
 	}
 
 	@Override
     public void reportContextSensitivity(@NotNull Parser recognizer, @NotNull DFA dfa,
-                                         int startIndex, int stopIndex, @NotNull OrderedHashSet<ATNConfig> configs)
+                                         int startIndex, int stopIndex, @NotNull ATNConfigSet configs)
     {
     }
 
@@ -579,7 +579,7 @@ public void reportInsufficientPredicates(@NotNull Parser recognizer,
 											 @NotNull IntervalSet ambigAlts,
 											 DecisionState decState,
 											 @NotNull SemanticContext[] altToPred,
-											 @NotNull OrderedHashSet<ATNConfig> configs, boolean fullContextParse)
+											 @NotNull ATNConfigSet configs, boolean fullContextParse)
     {
     }
 }

runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorStrategy.java

@@ -30,6 +30,7 @@ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.atn.ATNConfig;
+import org.antlr.v4.runtime.atn.ATNConfigSet;
 import org.antlr.v4.runtime.atn.DecisionState;
 import org.antlr.v4.runtime.atn.SemanticContext;
 import org.antlr.v4.runtime.dfa.DFA;
@@ -43,7 +44,7 @@ public class DiagnosticErrorStrategy extends DefaultErrorStrategy {
     @Override
     public void reportAmbiguity(@NotNull Parser recognizer,
 								DFA dfa, int startIndex, int stopIndex, @NotNull IntervalSet ambigAlts,
-								@NotNull OrderedHashSet<ATNConfig> configs)
+								@NotNull ATNConfigSet configs)
     {
         recognizer.notifyErrorListeners("reportAmbiguity d=" + dfa.decision + ": ambigAlts=" + ambigAlts + ":" + configs + ", input='" +
 										recognizer.getInputString(startIndex, stopIndex) + "'");
@@ -53,15 +54,15 @@ public void reportAmbiguity(@NotNull Parser recognizer,
 	public void reportAttemptingFullContext(@NotNull Parser recognizer,
 											@NotNull DFA dfa,
 											int startIndex, int stopIndex,
-											@NotNull OrderedHashSet<ATNConfig> configs)
+											@NotNull ATNConfigSet configs)
 	{
 		recognizer.notifyErrorListeners("reportAttemptingFullContext d=" + dfa.decision + ": " + configs + ", input='" +
 										recognizer.getInputString(startIndex, stopIndex) + "'");
 	}
 
 	@Override
 	public void reportContextSensitivity(@NotNull Parser recognizer, @NotNull DFA dfa,
-                                         int startIndex, int stopIndex, @NotNull OrderedHashSet<ATNConfig> configs)
+                                         int startIndex, int stopIndex, @NotNull ATNConfigSet configs)
     {
         recognizer.notifyErrorListeners("reportContextSensitivity d=" + dfa.decision + ": " + configs + ", input='" +
 										recognizer.getInputString(startIndex, stopIndex) + "'");
@@ -74,7 +75,7 @@ public void reportInsufficientPredicates(@NotNull Parser recognizer,
 											 @NotNull IntervalSet ambigAlts,
 											 DecisionState decState,
 											 @NotNull SemanticContext[] altToPred,
-											 @NotNull OrderedHashSet<ATNConfig> configs, boolean fullContextParse)
+											 @NotNull ATNConfigSet configs, boolean fullContextParse)
     {
         recognizer.notifyErrorListeners("reportInsufficientPredicates d=" + dfa.decision + ", decState=" + decState +
 										", ambigAlts=" + ambigAlts + ":" + Arrays.toString(altToPred) +

runtime/Java/src/org/antlr/v4/runtime/Lexer.java

@@ -88,7 +88,7 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
 	/** The token type for the current token */
 	public int _type;
 
-	public ArrayDeque<Integer> _modeStack;
+	public ArrayDeque<Integer> _modeStack = new ArrayDeque<Integer>();
 	public int _mode = Lexer.DEFAULT_MODE;
 
 	/** You can set the text for the current token to override what is in
@@ -115,9 +115,7 @@ public void reset() {
 
 		_hitEOF = false;
 		_mode = Lexer.DEFAULT_MODE;
-		if (_modeStack != null) {
-			_modeStack.clear();
-		}
+		_modeStack.clear();
 
 		getInterpreter().reset();
 	}
@@ -184,14 +182,13 @@ public void mode(int m) {
 
 	public void pushMode(int m) {
 		if ( LexerATNSimulator.debug ) System.out.println("pushMode "+m);
-		if ( _modeStack ==null ) _modeStack = new ArrayDeque<Integer>();
 		getInterpreter().tracePushMode(m);
 		_modeStack.push(_mode);
 		mode(m);
 	}
 
 	public int popMode() {
-		if ( _modeStack ==null ) throw new EmptyStackException();
+		if ( _modeStack.isEmpty() ) throw new EmptyStackException();
 		if ( LexerATNSimulator.debug ) System.out.println("popMode back to "+ _modeStack.peek());
 		getInterpreter().tracePopMode();
 		mode( _modeStack.pop() );

runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java

@@ -29,20 +29,19 @@ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 
 package org.antlr.v4.runtime;
 
-import org.antlr.v4.runtime.atn.ATNConfig;
-import org.antlr.v4.runtime.misc.OrderedHashSet;
+import org.antlr.v4.runtime.atn.ATNConfigSet;
 
 public class LexerNoViableAltException extends RecognitionException {
 	/** Matching attempted at what input index? */
 	public int startIndex;
 
 	/** Which configurations did we try at input.index() that couldn't match input.LA(1)? */
-	public OrderedHashSet<ATNConfig> deadEndConfigs;
+	public ATNConfigSet deadEndConfigs;
 
 	public LexerNoViableAltException(Lexer lexer,
 									 CharStream input,
 									 int startIndex,
-									 OrderedHashSet<ATNConfig> deadEndConfigs) {
+									 ATNConfigSet deadEndConfigs) {
 		super(lexer, input, null);
 		this.startIndex = startIndex;
 		this.deadEndConfigs = deadEndConfigs;