Port DH-12148: Formula Array Access

engine/api/src/main/java/io/deephaven/engine/table/Table.java

@@ -123,6 +123,8 @@ public interface Table extends
  String FILTERABLE_COLUMNS_ATTRIBUTE = "FilterableColumns";
  String TOTALS_TABLE_ATTRIBUTE = "TotalsTable";
  String ADD_ONLY_TABLE_ATTRIBUTE = "AddOnly";
+ String APPEND_ONLY_TABLE_ATTRIBUTE = "AppendOnly";
+ String TEST_SOURCE_TABLE_ATTRIBUTE = "TestSource";
  /**
  * <p>
  * If this attribute is present with value {@code true}, this Table is a "stream table".

engine/rowset/src/main/java/io/deephaven/engine/rowset/RowSetShiftData.java

@@ -305,10 +305,12 @@ public boolean apply(final WritableRowSet rowSet) {
  break;
  }
 
- // TODO: This loop is unfortunate, we will iterate the entire Index shift data; even if we have an input
- // index that is only a small subset. For the ending condition we solve that with the advance breaking
- // out of the loop, but for the starting condition, we can do better by binary searching the shift data
- // for the beginning of the index if the end of that range is less than the data.
+ // TODO #3341: This loop is unfortunate, we will iterate the entire RowSetShiftData; even if we have an
+ // input rowSet that is only a small subset. For the ending condition we solve that with the advance
+ // breaking out of the loop, but for the starting condition, we can do better by binary searching the
+ // shift data for the beginning of the index if the end of that range is less than the data. We can
+ // binary search for the next relevant shifted range anytime we attempt a shift that does not effect the
+ // rowSet.
  if (endRange < rsIt.peekNextKey()) {
  continue;
  }

engine/rowset/src/main/java/io/deephaven/engine/rowset/impl/WritableRowSetImpl.java

@@ -43,7 +43,8 @@ public WritableRowSetImpl(final OrderedLongSet innerSet) {
  this.innerSet = Objects.requireNonNull(innerSet);
  }
 
- protected final OrderedLongSet getInnerSet() {
+ @VisibleForTesting
+ public final OrderedLongSet getInnerSet() {
  return innerSet;
  }
 

engine/table/src/main/java/io/deephaven/engine/table/impl/AggAllByCopyAttributes.java

@@ -4,7 +4,6 @@
 package io.deephaven.engine.table.impl;
 
 import io.deephaven.api.agg.spec.*;
-import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.impl.BaseTable.CopyAttributeOperation;
 
 import java.util.Objects;

engine/table/src/main/java/io/deephaven/engine/table/impl/BaseTable.java

@@ -271,8 +271,14 @@ public enum CopyAttributeOperation {
 
  tempMap.put(ADD_ONLY_TABLE_ATTRIBUTE, EnumSet.of(
  CopyAttributeOperation.DropColumns,
- CopyAttributeOperation.UpdateView,
- CopyAttributeOperation.View,
+ CopyAttributeOperation.PartitionBy,
+ CopyAttributeOperation.Coalesce));
+
+
+ tempMap.put(APPEND_ONLY_TABLE_ATTRIBUTE, EnumSet.of(
+ CopyAttributeOperation.DropColumns,
+ CopyAttributeOperation.FirstBy,
+ CopyAttributeOperation.Flatten,
  CopyAttributeOperation.PartitionBy,
  CopyAttributeOperation.Coalesce));
 
@@ -367,7 +373,24 @@ public boolean isAddOnly() {
  if (!isRefreshing()) {
  return true;
  }
- return Boolean.TRUE.equals(getAttribute(Table.ADD_ONLY_TABLE_ATTRIBUTE));
+ boolean addOnly = Boolean.TRUE.equals(getAttribute(Table.ADD_ONLY_TABLE_ATTRIBUTE));
+ boolean appendOnly = Boolean.TRUE.equals(getAttribute(Table.APPEND_ONLY_TABLE_ATTRIBUTE));
+ return addOnly || appendOnly;
+ }
+
+ /**
+ * Returns true if this table is append-only, or has an attribute asserting that no modifies, shifts, or removals
+ * are generated and that all new rows are added to the end of the table.
+ *
+ * @return true if this table may only append rows at the end of the table
+ */
+ public boolean isAppendOnly() {
+ if (!isRefreshing()) {
+ return true;
+ }
+ boolean addOnly = Boolean.TRUE.equals(getAttribute(Table.ADD_ONLY_TABLE_ATTRIBUTE));
+ boolean appendOnly = Boolean.TRUE.equals(getAttribute(Table.APPEND_ONLY_TABLE_ATTRIBUTE));
+ return appendOnly || (addOnly && isFlat());
  }
 
  /**
@@ -594,11 +617,15 @@ public final void notifyListeners(final TableUpdate update) {
  if (isFlat()) {
  Assert.assertion(getRowSet().isFlat(), "build().isFlat()", getRowSet(), "build()");
  }
- if (isAddOnly()) {
+ if (isAppendOnly() || isAddOnly()) {
  Assert.assertion(update.removed().isEmpty(), "update.removed.empty()");
  Assert.assertion(update.modified().isEmpty(), "update.modified.empty()");
  Assert.assertion(update.shifted().empty(), "update.shifted.empty()");
  }
+ if (isAppendOnly()) {
+ Assert.assertion(getRowSet().lastRowKeyPrev() < update.added().firstRowKey(),
+ "getRowSet().lastRowKeyPrev() < update.added().firstRowKey()");
+ }
 
  // First validate that each rowSet is in a sane state.
  if (VALIDATE_UPDATE_INDICES) {

engine/table/src/main/java/io/deephaven/engine/table/impl/NaturalJoinHelper.java

@@ -39,6 +39,14 @@ static Table naturalJoin(QueryTable leftTable, QueryTable rightTable, MatchPair[
  naturalJoinInternal(leftTable, rightTable, columnsToMatch, columnsToAdd, exactMatch, control);
  leftTable.maybeCopyColumnDescriptions(result, rightTable, columnsToMatch, columnsToAdd);
  leftTable.copyAttributes(result, BaseTable.CopyAttributeOperation.Join);
+ // note in exact match we require that the right table can match as soon as a row is added to the left
+ boolean rightDoesNotGenerateModifies = !rightTable.isRefreshing() || (exactMatch && rightTable.isAddOnly());
+ if (leftTable.isAddOnly() && rightDoesNotGenerateModifies) {
+ result.setAttribute(Table.ADD_ONLY_TABLE_ATTRIBUTE, true);
+ }
+ if (leftTable.isAppendOnly() && rightDoesNotGenerateModifies) {
+ result.setAttribute(Table.APPEND_ONLY_TABLE_ATTRIBUTE, true);
+ }
  return result;
  }