Skip to content

Commit

Permalink
GH-43643: [Java] LargeListViewVector IPC Integration (#43681)
Browse files Browse the repository at this point in the history 
### Rationale for this change

Newly introduced `LargeListViewVector` requires the IPC integration for C Data integration tests while mainly supporting IPC format to include this type. 

### What changes are included in this PR?

Includes the `JsonFileWriter` and `JsonFileReader` along with the corresponding test cases. 

### Are these changes tested?

Yes, using existing tests but adding new configurations. 

### Are there any user-facing changes?

No
* GitHub Issue: #43643

Authored-by: Vibhatha Abeykoon <vibhatha@gmail.com>
Signed-off-by: David Li <li.davidm96@gmail.com>
  • Loading branch information
@vibhatha
vibhatha authored Aug 14, 2024
1 parent ce251a6 commit 712cfe6
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 4 deletions.
9 changes: 7 additions & 2 deletions java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.LargeListView;
import org.apache.arrow.vector.types.pojo.ArrowType.ListView;
import org.apache.arrow.vector.types.pojo.ArrowType.Union;
import org.apache.arrow.vector.types.pojo.Field;
Expand Down Expand Up @@ -729,7 +730,8 @@ private List<ArrowBuf> readIntoBuffer(
} else if (bufferType.equals(OFFSET) || bufferType.equals(SIZE)) {
if (type == MinorType.LARGELIST
|| type == MinorType.LARGEVARCHAR
|| type == MinorType.LARGEVARBINARY) {
|| type == MinorType.LARGEVARBINARY
|| type == MinorType.LARGELISTVIEW) {
reader = helper.INT8;
} else {
reader = helper.INT4;
Expand Down Expand Up @@ -890,7 +892,10 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws IOEx
BufferType bufferType = vectorTypes.get(v);
nextFieldIs(bufferType.getName());
int innerBufferValueCount = valueCount;
if (bufferType.equals(OFFSET) && !(type instanceof Union) && !(type instanceof ListView)) {
if (bufferType.equals(OFFSET)
&& !(type instanceof Union)
&& !(type instanceof ListView)
&& !(type instanceof LargeListView)) {
/* offset buffer has 1 additional value capacity except for dense unions and ListView */
innerBufferValueCount = valueCount + 1;
}
Expand Down
16 changes: 14 additions & 2 deletions java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
import org.apache.arrow.vector.UInt4Vector;
import org.apache.arrow.vector.UInt8Vector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.complex.BaseLargeRepeatedValueViewVector;
import org.apache.arrow.vector.complex.BaseRepeatedValueViewVector;
import org.apache.arrow.vector.dictionary.Dictionary;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
Expand Down Expand Up @@ -232,7 +233,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
final int bufferValueCount =
(bufferType.equals(OFFSET)
&& vector.getMinorType() != MinorType.DENSEUNION
&& vector.getMinorType() != MinorType.LISTVIEW)
&& vector.getMinorType() != MinorType.LISTVIEW
&& vector.getMinorType() != MinorType.LARGELISTVIEW)
? valueCount + 1
: valueCount;
for (int i = 0; i < bufferValueCount; i++) {
Expand Down Expand Up @@ -274,6 +276,7 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
} else if (bufferType.equals(OFFSET)
&& vector.getValueCount() == 0
&& (vector.getMinorType() == MinorType.LARGELIST
|| vector.getMinorType() == MinorType.LARGELISTVIEW
|| vector.getMinorType() == MinorType.LARGEVARBINARY
|| vector.getMinorType() == MinorType.LARGEVARCHAR)) {
// Empty vectors may not have allocated an offsets buffer
Expand Down Expand Up @@ -427,6 +430,10 @@ private void writeValueToGenerator(
generator.writeNumber(
buffer.getInt((long) index * BaseRepeatedValueViewVector.OFFSET_WIDTH));
break;
case LARGELISTVIEW:
generator.writeNumber(
buffer.getInt((long) index * BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
break;
case LARGELIST:
case LARGEVARBINARY:
case LARGEVARCHAR:
Expand Down Expand Up @@ -582,7 +589,12 @@ private void writeValueToGenerator(
throw new UnsupportedOperationException("minor type: " + vector.getMinorType());
}
} else if (bufferType.equals(SIZE)) {
generator.writeNumber(buffer.getInt((long) index * BaseRepeatedValueViewVector.SIZE_WIDTH));
if (vector.getMinorType() == MinorType.LISTVIEW) {
generator.writeNumber(buffer.getInt((long) index * BaseRepeatedValueViewVector.SIZE_WIDTH));
} else {
generator.writeNumber(
buffer.getInt((long) index * BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
}
}
}

Expand Down
8 changes: 8 additions & 0 deletions java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,10 +437,18 @@ public void testRoundtripEmptyVector() throws Exception {
"list",
FieldType.nullable(ArrowType.List.INSTANCE),
Collections.singletonList(Field.nullable("items", new ArrowType.Int(32, true)))),
new Field(
"listview",
FieldType.nullable(ArrowType.ListView.INSTANCE),
Collections.singletonList(Field.nullable("items", new ArrowType.Int(32, true)))),
new Field(
"largelist",
FieldType.nullable(ArrowType.LargeList.INSTANCE),
Collections.singletonList(Field.nullable("items", new ArrowType.Int(32, true)))),
new Field(
"largelistview",
FieldType.nullable(ArrowType.LargeListView.INSTANCE),
Collections.singletonList(Field.nullable("items", new ArrowType.Int(32, true)))),
new Field(
"map",
FieldType.nullable(new ArrowType.Map(/*keyssorted*/ false)),
Expand Down

0 comments on commit 712cfe6

Please sign in to comment.