Improve query library performance by using vector iteration (#3659)

* Update function libraries to use Vector iteration wherever possible

* Use Vector.isEmpty where possible.

* Address ambiguity of Vector.toArray() regarding mutation safety with additional documentation and a new method Vector.copyToArray()

* Address incorrect targetExclude in Base/build.gradle, and commit ring buffer code in sync with replication

* Use Vector.copyToArray() where appropriate in function libraries, and a few other related cleanups

* Remove PrimitiveType and ValueType for Boolean, and associate special casing in ftl files

---------

Co-authored-by: Ryan Caudy <rcaudy@gmail.com>

Base/build.gradle

@@ -48,7 +48,7 @@ artifacts {
 spotless {
  java {
  targetExclude(
- 'src/test/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBufferTest.java'
+ '**/ringbuffer/Aggregating*RingBuffer*.java'
  )
  }
 }

Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBuffer.java

@@ -696,8 +696,7 @@ void fixupTree(final long r1Head, final long r1Tail, final long r2Head, final lo
  * <p>
  * The provided ranges ore closed-interval. The head and tail are both included in the range.
  */
- private void evaluateThreeRanges(int r1h, int r1t, int r2h, int r2t, int r3h, int r3t,
- DoubleFunction evalFunction) {
+ private void evaluateThreeRanges(int r1h, int r1t, int r2h, int r2t, int r3h, int r3t, DoubleFunction evalFunction) {
  while (true) {
  if (r1t >= r2h) {
  // r1 and r2 overlap. Collapse them together and call the two range version.

Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingObjectRingBuffer.java

@@ -696,8 +696,7 @@ void fixupTree(final long r1Head, final long r1Tail, final long r2Head, final lo
  * <p>
  * The provided ranges ore closed-interval. The head and tail are both included in the range.
  */
- private void evaluateThreeRanges(int r1h, int r1t, int r2h, int r2t, int r3h, int r3t,
- ObjectFunction<T> evalFunction) {
+ private void evaluateThreeRanges(int r1h, int r1t, int r2h, int r2t, int r3h, int r3t, ObjectFunction<T> evalFunction) {
  while (true) {
  if (r1t >= r2h) {
  // r1 and r2 overlap. Collapse them together and call the two range version.

codegen/src/main/java/io/deephaven/generator/primitivetemplate/PrimitiveType.java

@@ -14,6 +14,8 @@ public class PrimitiveType {
  private final String boxed;
  private final String vector;
  private final String vectorDirect;
+ private final String vectorIterator;
+ private final String iteratorNext;
  private final String nullValue;
  private final String maxValue;
  private final String minValue;
@@ -26,17 +28,22 @@ public class PrimitiveType {
  * @param boxed name of the boxed primitive
  * @param vector name of the Vector interface
  * @param vectorDirect name of the concrete Vector wrapper
+ * @param vectorIterator name of the Vector iterator
+ * @param iteratorNext name of the next method on the Vector iterator
  * @param nullValue null value
  * @param maxValue maximum value
  * @param minValue minimum value
  * @param valueType type of value
  */
  public PrimitiveType(final String primitive, final String boxed, final String vector, final String vectorDirect,
+ final String vectorIterator, final String iteratorNext,
  final String nullValue, final String maxValue, final String minValue, final ValueType valueType) {
  this.primitive = primitive;
  this.boxed = boxed;
  this.vector = vector;
  this.vectorDirect = vectorDirect;
+ this.vectorIterator = vectorIterator;
+ this.iteratorNext = iteratorNext;
  this.nullValue = nullValue;
  this.maxValue = maxValue;
  this.minValue = minValue;
@@ -79,6 +86,24 @@ public String getVectorDirect() {
  return vectorDirect;
  }
 
+ /**
+ * Gets the Vector iterator of the primitive.
+ *
+ * @return Vector iterator of the primitive.
+ */
+ public String getVectorIterator() {
+ return vectorIterator;
+ }
+
+ /**
+ * Gets the next method on the Vector iterator of the primitive.
+ *
+ * @return next method on the Vector iterator of the primitive.
+ */
+ public String getIteratorNext() {
+ return iteratorNext;
+ }
+
  /**
  * Gets the null value of the primitive.
  *
@@ -132,36 +157,39 @@ public int hashCode() {
 
  public static PrimitiveType[] primitiveTypes() {
  return new PrimitiveType[] {
- new PrimitiveType("Boolean", "Boolean",
- "ObjectVector", "ObjectVectorDirect",
- "NULL_BOOLEAN", null, null,
- ValueType.BOOLEAN),
  new PrimitiveType("char", "Character",
  "CharVector", "CharVectorDirect",
+ "CloseablePrimitiveIteratorOfChar", "nextChar",
  "NULL_CHAR", null, null,
  ValueType.CHARACTER),
  new PrimitiveType("byte", "Byte",
  "ByteVector", "ByteVectorDirect",
+ "CloseablePrimitiveIteratorOfByte", "nextByte",
  "NULL_BYTE", "MAX_BYTE", "MIN_BYTE",
  ValueType.INTEGER),
  new PrimitiveType("short", "Short",
  "ShortVector", "ShortVectorDirect",
+ "CloseablePrimitiveIteratorOfShort", "nextShort",
  "NULL_SHORT", "MAX_SHORT", "MIN_SHORT",
  ValueType.INTEGER),
  new PrimitiveType("int", "Integer",
  "IntVector", "IntVectorDirect",
+ "CloseablePrimitiveIteratorOfInt", "nextInt",
  "NULL_INT", "MAX_INT", "MIN_INT",
  ValueType.INTEGER),
  new PrimitiveType("long", "Long",
  "LongVector", "LongVectorDirect",
+ "CloseablePrimitiveIteratorOfLong", "nextLong",
  "NULL_LONG", "MAX_LONG", "MIN_LONG",
  ValueType.INTEGER),
  new PrimitiveType("float", "Float",
  "FloatVector", "FloatVectorDirect",
+ "CloseablePrimitiveIteratorOfFloat", "nextFloat",
  "NULL_FLOAT", "MAX_FLOAT", "MIN_FLOAT",
  ValueType.FLOATING_POINT),
  new PrimitiveType("double", "Double",
  "DoubleVector", "DoubleVectorDirect",
+ "CloseablePrimitiveIteratorOfDouble", "nextDouble",
  "NULL_DOUBLE", "MAX_DOUBLE", "MIN_DOUBLE",
  ValueType.FLOATING_POINT),
  };

codegen/src/main/java/io/deephaven/generator/primitivetemplate/ValueType.java

@@ -7,35 +7,23 @@
  * Type of a value.
  */
 public enum ValueType {
- BOOLEAN(true, false, false, false),
 
- CHARACTER(false, true, false, false),
+ CHARACTER(true, false, false),
 
- INTEGER(false, false, true, false),
+ INTEGER(false, true, false),
 
- FLOATING_POINT(false, false, false, true);
+ FLOATING_POINT(false, false, true);
 
- private final boolean isBoolean;
  private final boolean isChar;
  private final boolean isInteger;
  private final boolean isFloat;
 
- ValueType(boolean isBoolean, boolean isChar, boolean isInteger, boolean isFloat) {
- this.isBoolean = isBoolean;
+ ValueType(boolean isChar, boolean isInteger, boolean isFloat) {
  this.isChar = isChar;
  this.isInteger = isInteger;
  this.isFloat = isFloat;
  }
 
- /**
- * Is the value a boolean.
- *
- * @return true for booleans, and false otherwise.
- */
- public boolean getIsBoolean() {
- return isBoolean;
- }
-
  /**
  * Is the value a character.
  *

codegen/src/test/java/io/deephaven/generator/primitivetemplate/TestPrimitiveType.java

@@ -12,13 +12,16 @@ public void testPrimitiveType() {
  final PrimitiveType pt = new PrimitiveType(
  "primitive", "boxed",
  "VectorDEBUG", "DirectDEBUG",
+ "IteratorDEBUG", "IteratorNextDEBUG",
  "NULLDEBUG", "POSINFDEBUG", "NEGINFDEBUG",
  ValueType.CHARACTER);
 
  assertEquals("primitive", pt.getPrimitive());
  assertEquals("boxed", pt.getBoxed());
  assertEquals("VectorDEBUG", pt.getVector());
  assertEquals("DirectDEBUG", pt.getVectorDirect());
+ assertEquals("IteratorDEBUG", pt.getVectorIterator());
+ assertEquals("IteratorNextDEBUG", pt.getIteratorNext());
  assertEquals("NULLDEBUG", pt.getNull());
  assertEquals("POSINFDEBUG", pt.getMaxValue());
  assertEquals("NEGINFDEBUG", pt.getMinValue());

engine/function/build.gradle

@@ -16,6 +16,7 @@ dependencies {
  api 'net.sf.trove4j:trove4j:3.0.3'
  api project(':Util')
  api project(':engine-vector')
+ api project(':engine-primitive')
 
  implementation project(':Base')
  implementation depCommonsLang3

engine/function/src/main/java/io/deephaven/function/Logic.java

@@ -4,6 +4,7 @@
 package io.deephaven.function;
 
 import io.deephaven.vector.ObjectVector;
+import io.deephaven.engine.primitive.iterator.*;
 
 /**
  * Logic functions.
@@ -49,10 +50,12 @@ static public Boolean and(boolean... values) {
  * @return logical and of all the values in the array. By convention, returns true if the array is empty.
  */
  static public Boolean and(ObjectVector<Boolean> values) {
- for (int ii = 0; ii < values.size(); ++ii) {
- Boolean b = values.get(ii);
- if (!b) {
- return false;
+ try (final CloseableIterator<Boolean> vi = values.iterator()) {
+ while (vi.hasNext()) {
+ final Boolean b = vi.next();
+ if (!b) {
+ return false;
+ }
  }
  }
 
@@ -86,12 +89,14 @@ static public Boolean and(Boolean[] values, Boolean nullValue) {
  * @return logical and of all the values in the array. By convention, returns true if the array is empty.
  */
  static public Boolean and(ObjectVector<Boolean> values, Boolean nullValue) {
- for (int ii = 0; ii < values.size(); ++ii) {
- Boolean b = values.get(ii);
- b = b == null ? nullValue : b;
-
- if (!b) {
- return false;
+ try (final CloseableIterator<Boolean> vi = values.iterator()) {
+ while (vi.hasNext()) {
+ final Boolean b = vi.next();
+ final Boolean b2 = b == null ? nullValue : b;
+
+ if (!b2) {
+ return false;
+ }
  }
  }