Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-41569: [Java] ListViewVector Implementation for UnionListViewReader #43077

Merged
merged 37 commits into from
Aug 1, 2024
Merged
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
fa7bbf4
fix: upgrade to junit5
vibhatha Jun 6, 2024
815de1e
feat: initial partial commit
vibhatha Jun 7, 2024
96f82f8
feat: adding rest of the test cases: snapshot 1: requires transferPai…
vibhatha Jun 7, 2024
3ff5a85
fix: adding intermediate transferPair function
vibhatha Jun 11, 2024
1c73d48
fix: refactor utils
vibhatha Jun 11, 2024
681ee28
fix: addressing split and transfer initial release
vibhatha Jun 11, 2024
260cf5d
fix: refactor util locations
vibhatha Jun 28, 2024
c5daa57
fix: rebase
vibhatha Jul 2, 2024
a63728b
temp
vibhatha Jul 3, 2024
33222bd
fix: revert ListViewWriter to ListWriter
vibhatha Jul 3, 2024
42763ed
fix: temp view fix testing
vibhatha Jul 4, 2024
c023634
fix: adding minor changes to the interfaces
vibhatha Jul 4, 2024
e60caa8
fix: bug in promotion logic
vibhatha Jul 8, 2024
2edffda
fix: adding startListView and endListView usage
vibhatha Jul 9, 2024
a3b93d7
fix: updating test cases for new interface
vibhatha Jul 9, 2024
a43d8fc
fix: updating test cases
vibhatha Jul 9, 2024
692675e
fix: adding json reader/writer
vibhatha Jul 10, 2024
00efc88
fix: adding test cases
vibhatha Jul 10, 2024
febd540
fix: adding c data component v1 WIP
vibhatha Jul 11, 2024
e181e6a
feat: adding rountrip tests for c data interface
vibhatha Jul 22, 2024
df778a2
feat: adding java/python integration tests v1
vibhatha Jul 22, 2024
3e8815a
fix: minor typo
vibhatha Jul 22, 2024
059a2a8
feat: adding listview integration tests
vibhatha Jul 22, 2024
2d8b412
fix: adding range equal visitor test cases
vibhatha Jul 22, 2024
e6da575
fix: adding type equal visitor tests
vibhatha Jul 22, 2024
69b5742
fix: review
vibhatha Jul 23, 2024
9f4c9ff
fix: minor test option
vibhatha Jul 23, 2024
05b0998
fix: add new line
vibhatha Jul 23, 2024
dbfd228
fix: removing C Data interface components
vibhatha Jul 24, 2024
cf5ebcc
fix: addressing reviews v1
vibhatha Jul 24, 2024
e7daa32
fix: addressing reviews v2
vibhatha Jul 24, 2024
0561213
fix: removing visitor related component
vibhatha Jul 25, 2024
7603db6
fix: remove setters and inline
vibhatha Jul 29, 2024
3fe3f3f
fix: removing size=0 case for listview
vibhatha Jul 29, 2024
25c01a5
fix: revert moving logic to sub-class
vibhatha Jul 30, 2024
32dd3fc
fix: addressing reviews v3
vibhatha Aug 1, 2024
3cc972e
fix: format
vibhatha Aug 1, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: addressing reviews v2
vibhatha committed Aug 1, 2024

Verified

This commit was signed with the committer’s verified signature. The key has expired.
kylekurz Kyle Kurz
commit e7daa3238b3007b841817023d3a47451132eb0f3
Original file line number Diff line number Diff line change
@@ -81,10 +81,10 @@ public PromotableViewWriter(ValueVector v, AbstractStructVector parentContainer,
protected FieldWriter getWriter(MinorType type, ArrowType arrowType) {
if (state == State.UNION) {
if (requiresArrowType(type)) {
writer = ((UnionWriter) writer).promote();
writer = ((UnionWriter) writer).toViewWriter();
((UnionViewWriter) writer).getWriter(type, arrowType);
} else {
writer = ((UnionWriter) writer).promote();
writer = ((UnionWriter) writer).toViewWriter();
((UnionViewWriter) writer).getWriter(type);
}
} else if (state == State.UNTYPED) {
@@ -112,10 +112,10 @@ protected FieldWriter getWriter(MinorType type, ArrowType arrowType) {
} else if (type != this.type) {
promoteToUnion();
if (requiresArrowType(type)) {
writer = ((UnionWriter) writer).promote();
writer = ((UnionWriter) writer).toViewWriter();
((UnionViewWriter) writer).getWriter(type, arrowType);
} else {
writer = ((UnionWriter) writer).promote();
writer = ((UnionWriter) writer).toViewWriter();
((UnionViewWriter) writer).getWriter(type);
}
}
@@ -140,7 +140,6 @@ public StructWriter struct() {

</#list></#list>


@Override
public void allocate() {
getWriter().allocate();
19 changes: 10 additions & 9 deletions java/vector/src/main/codegen/templates/PromotableWriter.java
Original file line number Diff line number Diff line change
@@ -539,27 +539,28 @@ public void close() throws Exception {
getWriter().close();
}

public void setState(State state) {
protected void setState(State state) {
this.state = state;
}

public void setType(MinorType type) {
protected void setType(MinorType type) {
this.type = type;
}

public void setUnionVector(UnionVector unionVector) {
protected void setUnionVector(UnionVector unionVector) {
this.unionVector = unionVector;
}

public void setVector(ValueVector vector) {
this.vector = vector;
}

public void setWriter(FieldWriter writer) {
protected void setWriter(FieldWriter writer) {
this.writer = writer;
}

public PromotableViewWriter promote() {
/**
* Convert the writer to a PromotableViewWriter.
*
* @return The writer as a PromotableViewWriter.
*/
public PromotableViewWriter toViewWriter() {
PromotableViewWriter promotableViewWriter = new PromotableViewWriter(unionVector, parentContainer, nullableStructWriterFactory);
promotableViewWriter.setPosition(position);
promotableViewWriter.setWriter(writer);
Original file line number Diff line number Diff line change
@@ -221,7 +221,7 @@ public ListWriter listView(String name) {
// ensure writers are initialized
((PromotableViewWriter) writer).getWriter(MinorType.LISTVIEW);
} else {
writer = ((PromotableWriter) writer).promote();
writer = ((PromotableWriter) writer).toViewWriter();
((PromotableViewWriter) writer).getWriter(MinorType.LISTVIEW);
}
}
21 changes: 11 additions & 10 deletions java/vector/src/main/codegen/templates/UnionWriter.java
Original file line number Diff line number Diff line change
@@ -59,7 +59,12 @@ public UnionWriter(UnionVector vector, NullableStructWriterFactory nullableStruc
this.nullableStructWriterFactory = nullableStructWriterFactory;
}

public UnionViewWriter promote() {
/**
* Convert the UnionWriter to a UnionViewWriter.
*
* @return the converted UnionViewWriter
*/
public UnionViewWriter toViewWriter() {
UnionViewWriter unionViewWriter = new UnionViewWriter(data, nullableStructWriterFactory);
unionViewWriter.setStructWriter(structWriter);
unionViewWriter.setListWriter(listWriter);
@@ -70,27 +75,23 @@ public UnionViewWriter promote() {
return unionViewWriter;
}

public void setStructWriter(StructWriter structWriter) {
protected void setStructWriter(StructWriter structWriter) {
this.structWriter = structWriter;
}

public void setListWriter(UnionListWriter listWriter) {
protected void setListWriter(UnionListWriter listWriter) {
this.listWriter = listWriter;
}

public void setListViewWriter(UnionListViewWriter listViewWriter) {
protected void setListViewWriter(UnionListViewWriter listViewWriter) {
this.listViewWriter = listViewWriter;
}

public void setMapWriter(UnionMapWriter mapWriter) {
protected void setMapWriter(UnionMapWriter mapWriter) {
this.mapWriter = mapWriter;
}

public List<BaseWriter> getWriters() {
return writers;
}

public void setWriters(List<BaseWriter> writers) {
protected void setWriters(List<BaseWriter> writers) {
this.writers = writers;
}

Original file line number Diff line number Diff line change
@@ -250,17 +250,7 @@ public List<ArrowBuf> getFieldBuffers() {
@Override
public void exportCDataBuffers(List<ArrowBuf> buffers, ArrowBuf buffersPtr, long nullValue) {
exportBuffer(validityBuffer, buffers, buffersPtr, nullValue, true);

if (offsetBuffer.capacity() == 0) {
// Empty offset buffer is allowed for historical reason.
// To export it through C Data interface, we need to allocate a buffer with one offset.
// We set `retain = false` to explicitly not increase the ref count for the exported buffer.
// The ref count of the newly created buffer (i.e., 1) already represents the usage
// at imported side.
exportBuffer(allocateBuffers(OFFSET_WIDTH), buffers, buffersPtr, nullValue, false);
} else {
exportBuffer(offsetBuffer, buffers, buffersPtr, nullValue, true);
}
exportBuffer(offsetBuffer, buffers, buffersPtr, nullValue, true);
exportBuffer(sizeBuffer, buffers, buffersPtr, nullValue, true);
}

Original file line number Diff line number Diff line change
@@ -891,7 +891,7 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws IOEx
nextFieldIs(bufferType.getName());
int innerBufferValueCount = valueCount;
if (bufferType.equals(OFFSET) && !(type instanceof Union) && !(type instanceof ListView)) {
/* offset buffer has 1 additional value capacity except for dense unions */
/* offset buffer has 1 additional value capacity except for dense unions and ListView */
innerBufferValueCount = valueCount + 1;
}