Replace edge representation in DFAState class with a fast hashmap (Java runtime).

runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestDFAEdgeCache.java

@@ -0,0 +1,185 @@
+package org.antlr.v4.test.runtime.java;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.antlr.v4.runtime.dfa.DFAState;
+import org.antlr.v4.runtime.dfa.DFAEdgeCache;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestDFAEdgeCache {
+
+	@Test
+	public void initializesCorrectly() {
+		// Check first 1K initial sizes.
+		for (int i = 1; i < 1000; i++) {
+			DFAEdgeCache im = new DFAEdgeCache(i);
+			checkSize(im, 0);
+		}
+	}
+
+	@Test
+	public void failsOnInvalidSizes() {
+		try {
+			DFAEdgeCache im;
+			im = new DFAEdgeCache(0);
+			im = new DFAEdgeCache(-1);
+			im = new DFAEdgeCache(Integer.MAX_VALUE);
+			im = new DFAEdgeCache(Integer.MIN_VALUE);
+			im = new DFAEdgeCache(1 << 29 + 1);
+			Assert.fail("Illegal size should have thrown an exception.");
+		} catch (RuntimeException e) {
+			// Nothing to do
+		}
+	}
+
+	@Test
+	public void expandsCorrectly() {
+		// Create maps with different sizes and add size * 10 elements to each.
+		for (int i = 1; i < 100; i++) {
+			DFAEdgeCache im = new DFAEdgeCache(i);
+			// Insert i * 10 elements to each and confirm sizes
+			int elements = i * 10;
+			for (int j = 0; j < elements; j++) {
+				im.addEdge(j, new DFAState(j));
+			}
+			for (int j = 0; j < elements; j++) {
+				assertStateEquals(j, im.getTargetState(j));
+			}
+			checkSize(im, elements);
+		}
+	}
+
+
+	@Test
+	public void putAddsAndUpdatesElementsCorrectly() {
+		int span = 100;
+		for (int i = 0; i < span; i++) {
+			DFAEdgeCache im = new DFAEdgeCache();
+			checkSpanInsertions(im, -i, i);
+		}
+		// Do the same, this time overwrite values as well
+		DFAEdgeCache im = new DFAEdgeCache();
+		for (int i = 0; i < span; i++) {
+			checkSpanInsertions(im, -i, i);
+			checkSpanInsertions(im, -i, i);
+			checkSpanInsertions(im, -i, i);
+		}
+	}
+
+	@Test
+	public void survivesSimpleFuzzing() {
+		List<int[]> fuzzLists = createFuzzingLists();
+		for (int[] arr : fuzzLists) {
+			DFAEdgeCache im = new DFAEdgeCache();
+			for (int i = 0; i < arr.length; i++) {
+				im.addEdge(arr[i], new DFAState(arr[i]));
+				assertStateEquals(arr[i], im.getTargetState(arr[i]));
+			}
+		}
+
+		DFAEdgeCache im = new DFAEdgeCache();
+		for (int[] arr : fuzzLists) {
+			for (int i = 0; i < arr.length; i++) {
+				im.addEdge(arr[i], new DFAState(arr[i]));
+				assertStateEquals(arr[i], im.getTargetState(arr[i]));
+			}
+		}
+	}
+
+	private List<int[]> createFuzzingLists() {
+		List<int[]> fuzzLists = new ArrayList<>(5000);
+		int maxListSize = 300;
+		Random r = new Random(0xBEEFCAFE);
+		// Random sized lists with values in [0..n] shuffled.
+		for (int i = 0; i < 1000; i++) {
+			int[] arr = new int[r.nextInt(maxListSize) + 1];
+			for (int j = 0; j < arr.length; j++) {
+				arr[j] = j;
+			}
+			shuffle(arr);
+			fuzzLists.add(arr);
+		}
+		// Random sized lists with values in [-n..n] shuffled.
+		for (int i = 0; i < 1000; i++) {
+			int size = r.nextInt(maxListSize) + 1;
+			int[] arr = new int[size * 2];
+			int idx = 0;
+			for (int j = 0; j < arr.length; j++) {
+				arr[idx++] = j - size;
+			}
+			shuffle(arr);
+			fuzzLists.add(arr);
+		}
+		// Random sized lists in [-m,m] shuffled. Possible duplicates.
+		int m = 1 << 10;
+		for (int i = 0; i < 2000; i++) {
+			int size = r.nextInt(maxListSize) + 1;
+			int[] arr = new int[size];
+			for (int j = 0; j < arr.length; j++) {
+				arr[j] = r.nextInt(2 * m) - m;
+			}
+			shuffle(arr);
+			fuzzLists.add(arr);
+		}
+		return fuzzLists;
+	}
+
+	private void checkSpanInsertions(DFAEdgeCache im, int start, int end) {
+		insertSpan(im, start, end);
+		// Expected size.
+		int size = Math.abs(start) + Math.abs(end) + 1;
+		assertEquals(size, im.size());
+		checkSpan(im, start, end);
+	}
+
+	private void insertSpan(DFAEdgeCache im, int start, int end) {
+		int spanStart = Math.min(start, end);
+		int spanEnd = Math.max(start, end);
+		for (int i = spanStart; i <= spanEnd; i++) {
+			im.addEdge(i, new DFAState(i));
+		}
+	}
+
+	private void checkSpan(DFAEdgeCache im, int start, int end) {
+		int spanStart = Math.min(start, end);
+		int spanEnd = Math.max(start, end);
+		for (int i = spanStart; i <= spanEnd; i++) {
+			assertStateEquals(i, im.getTargetState(i));
+		}
+		// Check outside of span values do not exist in the map
+		for (int i = spanStart - 1, idx = 0; idx < 100; i--, idx++) {
+			Assert.assertNull(im.getTargetState(i));
+		}
+		for (int i = spanEnd + 1, idx = 0; idx < 100; i++, idx++) {
+			Assert.assertNull(im.getTargetState(i));
+		}
+	}
+
+	private void checkSize(DFAEdgeCache m, int size) {
+		assertEquals(size, m.size());
+		assertTrue(m.capacity() > m.size());
+		// Check capacity is 2^n
+		assertTrue((m.capacity() & (m.capacity() - 1)) == 0);
+	}
+
+	// Fisher yates shuffle
+	private static void shuffle(int[] array) {
+		int index, temp;
+		Random random = new Random(0xCAFEBABE);
+		for (int i = array.length - 1; i > 0; i--) {
+			index = random.nextInt(i + 1);
+			temp = array[index];
+			array[index] = array[i];
+			array[i] = temp;
+		}
+	}
+
+	private void assertStateEquals(int symbol, DFAState state) {
+		Assert.assertEquals(symbol, state.stateNumber);
+	}
+}

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -22,9 +22,6 @@ public class LexerATNSimulator extends ATNSimulator {
 	public static final boolean debug = false;
 	public static final boolean dfa_debug = false;
 
-	public static final int MIN_DFA_EDGE = 0;
-	public static final int MAX_DFA_EDGE = 127; // forces unicode to stay in ATN
-
 	/** When we hit an accept state in either the DFA or the ATN, we
 	 *  have to notify the character stream to start buffering characters
 	 *  via {@link IntStream#mark} and record the current state. The current sim state
@@ -244,19 +241,8 @@ protected int execATN(CharStream input, DFAState ds0) {
 	 * {@code t}, or {@code null} if the target state for this edge is not
 	 * already cached
 	 */
-
 	protected DFAState getExistingTargetState(DFAState s, int t) {
-		if (s.edges == null || t < MIN_DFA_EDGE || t > MAX_DFA_EDGE) {
-			return null;
-		}
-
-		DFAState target = s.edges[t - MIN_DFA_EDGE];
-		if (debug && target != null) {
-			System.out.println("reuse state "+s.stateNumber+
-							   " edge to "+target.stateNumber);
-		}
-
-		return target;
+		return s.getTargetState(t);
 	}
 
 	/**
@@ -644,22 +630,10 @@ protected DFAState addDFAEdge(DFAState from,
 	}
 
 	protected void addDFAEdge(DFAState p, int t, DFAState q) {
-		if (t < MIN_DFA_EDGE || t > MAX_DFA_EDGE) {
-			// Only track edges within the DFA bounds
-			return;
-		}
-
 		if ( debug ) {
 			System.out.println("EDGE "+p+" -> "+q+" upon "+((char)t));
 		}
-
-		synchronized (p) {
-			if ( p.edges==null ) {
-				//  make room for tokens 1..n and -1 masquerading as index 0
-				p.edges = new DFAState[MAX_DFA_EDGE-MIN_DFA_EDGE+1];
-			}
-			p.edges[t - MIN_DFA_EDGE] = q; // connect
-		}
+		p.addEdge(t, q);
 	}
 
 	/** Add a new DFA state if there isn't one with this set of
@@ -668,13 +642,15 @@ protected void addDFAEdge(DFAState p, int t, DFAState q) {
 		traversing the DFA, we will know which rule to accept.
 	 */
 
+
 	protected DFAState addDFAState(ATNConfigSet configs) {
 		/* the lexer evaluates predicates on-the-fly; by this point configs
 		 * should not contain any configurations with unevaluated predicates.
 		 */
 		assert !configs.hasSemanticContext;
 
 		DFAState proposed = new DFAState(configs);
+
 		ATNConfig firstConfigWithRuleStopState = null;
 		for (ATNConfig c : configs) {
 			if ( c.state instanceof RuleStopState )	{
@@ -695,7 +671,6 @@ protected DFAState addDFAState(ATNConfigSet configs) {
 			if ( existing!=null ) return existing;
 
 			DFAState newState = proposed;
-
 			newState.stateNumber = dfa.states.size();
 			configs.setReadonly(true);
 			newState.configs = configs;
@@ -704,7 +679,6 @@ protected DFAState addDFAState(ATNConfigSet configs) {
 		}
 	}
 
-
 	public final DFA getDFA(int mode) {
 		return decisionToDFA[mode];
 	}

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -552,12 +552,7 @@ protected int execATN(DFA dfa, DFAState s0,
 	 * already cached
 	 */
 	protected DFAState getExistingTargetState(DFAState previousD, int t) {
-		DFAState[] edges = previousD.edges;
-		if (edges == null || t + 1 < 0 || t + 1 >= edges.length) {
-			return null;
-		}
-
-		return edges[t + 1];
+		return previousD.getTargetState(t);
 	}
 
 	/**
@@ -2062,10 +2057,7 @@ else if ( c.alt!=alt ) {
 	 * otherwise this method returns the result of calling {@link #addDFAState}
 	 * on {@code to}
 	 */
-	protected DFAState addDFAEdge(DFA dfa,
-								  DFAState from,
-								  int t,
-								  DFAState to)
+	protected DFAState addDFAEdge(DFA dfa, DFAState from, int t, DFAState to)
 	{
 		if ( debug ) {
 			System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
@@ -2075,18 +2067,12 @@ protected DFAState addDFAEdge(DFA dfa,
 			return null;
 		}
 
-		to = addDFAState(dfa, to); // used existing if possible not incoming
+		to = addDFAState(dfa, to); // Use existing if possible not incoming
 		if (from == null || t < -1 || t > atn.maxTokenType) {
 			return to;
 		}
 
-		synchronized (from) {
-			if ( from.edges==null ) {
-				from.edges = new DFAState[atn.maxTokenType+1+1];
-			}
-
-			from.edges[t+1] = to; // connect
-		}
+  	from.addEdge(t, to);
 
 		if ( debug ) {
 			System.out.println("DFA=\n"+dfa.toString(parser!=null?parser.getVocabulary():VocabularyImpl.EMPTY_VOCABULARY));
@@ -2095,6 +2081,7 @@ protected DFAState addDFAEdge(DFA dfa,
 		return to;
 	}
 
+
 	/**
 	 * Add state {@code D} to the DFA if it is not already present, and return
 	 * the actual instance stored in the DFA. If a state equivalent to {@code D}

runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java

@@ -13,7 +13,6 @@
 import org.antlr.v4.runtime.atn.StarLoopEntryState;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -55,7 +54,6 @@ public DFA(DecisionState atnStartState, int decision) {
 			if (((StarLoopEntryState)atnStartState).isPrecedenceDecision) {
 				precedenceDfa = true;
 				DFAState precedenceState = new DFAState(new ATNConfigSet());
-				precedenceState.edges = new DFAState[0];
 				precedenceState.isAcceptState = false;
 				precedenceState.requiresFullContext = false;
 				this.s0 = precedenceState;
@@ -68,7 +66,7 @@ public DFA(DecisionState atnStartState, int decision) {
 	/**
 	 * Gets whether this DFA is a precedence DFA. Precedence DFAs use a special
 	 * start state {@link #s0} which is not stored in {@link #states}. The
-	 * {@link DFAState#edges} array for this start state contains outgoing edges
+	 * {@link DFAState#edges} map for this start state contains outgoing edges
 	 * supplying individual start states corresponding to specific precedence
 	 * values.
 	 *
@@ -95,13 +93,7 @@ public final DFAState getPrecedenceStartState(int precedence) {
 		if (!isPrecedenceDfa()) {
 			throw new IllegalStateException("Only precedence DFAs may contain a precedence start state.");
 		}
-
-		// s0.edges is never null for a precedence DFA
-		if (precedence < 0 || precedence >= s0.edges.length) {
-			return null;
-		}
-
-		return s0.edges[precedence];
+		return s0.getTargetState(precedence);
 	}
 
 	/**
@@ -124,16 +116,7 @@ public final void setPrecedenceStartState(int precedence, DFAState startState) {
 			return;
 		}
 
-		// synchronization on s0 here is ok. when the DFA is turned into a
-		// precedence DFA, s0 will be initialized once and not updated again
-		synchronized (s0) {
-			// s0.edges is never null for a precedence DFA
-			if (precedence >= s0.edges.length) {
-				s0.edges = Arrays.copyOf(s0.edges, precedence + 1);
-			}
-
-			s0.edges[precedence] = startState;
-		}
+		s0.addEdge(precedence, startState);
 	}
 
 	/**