Skip to content

PARQUET-11: Reduce memory pressure when reading footers #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
julienledem wants to merge 3 commits into from
Closed

PARQUET-11: Reduce memory pressure when reading footers #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7664919
intern parquet metadata strings when reading them
dvryaboy Jul 3, 2014
1c382cc
implement delegate instead
julienledem Aug 20, 2014
96ff408
Merge branch 'master' into reduce_metadata_memory
julienledem Aug 20, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 215 additions & 0 deletions src/main/java/parquet/format/InterningProtocol.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
package parquet.format;

import java.nio.ByteBuffer;

import org.apache.thrift.TException;
import org.apache.thrift.protocol.TField;
import org.apache.thrift.protocol.TList;
import org.apache.thrift.protocol.TMap;
import org.apache.thrift.protocol.TMessage;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.protocol.TSet;
import org.apache.thrift.protocol.TStruct;
import org.apache.thrift.transport.TTransport;

/**
* TProtocol that interns the strings.
*
* @author Julien Le Dem
*
*/
public class InterningProtocol extends TProtocol {

private final TProtocol delegate;

public InterningProtocol(TProtocol delegate) {
super(delegate.getTransport());
this.delegate = delegate;
}

public TTransport getTransport() {
return delegate.getTransport();
}

public void writeMessageBegin(TMessage message) throws TException {
delegate.writeMessageBegin(message);
}

public void writeMessageEnd() throws TException {
delegate.writeMessageEnd();
}

public int hashCode() {
return delegate.hashCode();
}

public void writeStructBegin(TStruct struct) throws TException {
delegate.writeStructBegin(struct);
}

public void writeStructEnd() throws TException {
delegate.writeStructEnd();
}

public void writeFieldBegin(TField field) throws TException {
delegate.writeFieldBegin(field);
}

public void writeFieldEnd() throws TException {
delegate.writeFieldEnd();
}

public void writeFieldStop() throws TException {
delegate.writeFieldStop();
}

public void writeMapBegin(TMap map) throws TException {
delegate.writeMapBegin(map);
}

public void writeMapEnd() throws TException {
delegate.writeMapEnd();
}

public void writeListBegin(TList list) throws TException {
delegate.writeListBegin(list);
}

public void writeListEnd() throws TException {
delegate.writeListEnd();
}

public void writeSetBegin(TSet set) throws TException {
delegate.writeSetBegin(set);
}

public void writeSetEnd() throws TException {
delegate.writeSetEnd();
}

public void writeBool(boolean b) throws TException {
delegate.writeBool(b);
}

public void writeByte(byte b) throws TException {
delegate.writeByte(b);
}

public void writeI16(short i16) throws TException {
delegate.writeI16(i16);
}

public void writeI32(int i32) throws TException {
delegate.writeI32(i32);
}

public void writeI64(long i64) throws TException {
delegate.writeI64(i64);
}

public void writeDouble(double dub) throws TException {
delegate.writeDouble(dub);
}

public void writeString(String str) throws TException {
delegate.writeString(str);
}

public void writeBinary(ByteBuffer buf) throws TException {
delegate.writeBinary(buf);
}

public TMessage readMessageBegin() throws TException {
return delegate.readMessageBegin();
}

public void readMessageEnd() throws TException {
delegate.readMessageEnd();
}

public TStruct readStructBegin() throws TException {
return delegate.readStructBegin();
}

public void readStructEnd() throws TException {
delegate.readStructEnd();
}

public TField readFieldBegin() throws TException {
return delegate.readFieldBegin();
}

public void readFieldEnd() throws TException {
delegate.readFieldEnd();
}

public TMap readMapBegin() throws TException {
return delegate.readMapBegin();
}

public void readMapEnd() throws TException {
delegate.readMapEnd();
}

public TList readListBegin() throws TException {
return delegate.readListBegin();
}

public void readListEnd() throws TException {
delegate.readListEnd();
}

public TSet readSetBegin() throws TException {
return delegate.readSetBegin();
}

public void readSetEnd() throws TException {
delegate.readSetEnd();
}

public boolean equals(Object obj) {
return delegate.equals(obj);
}

public boolean readBool() throws TException {
return delegate.readBool();
}

public byte readByte() throws TException {
return delegate.readByte();
}

public short readI16() throws TException {
return delegate.readI16();
}

public int readI32() throws TException {
return delegate.readI32();
}

public long readI64() throws TException {
return delegate.readI64();
}

public double readDouble() throws TException {
return delegate.readDouble();
}

public String readString() throws TException {
// this is where we intern the strings
return delegate.readString().intern();
}

public ByteBuffer readBinary() throws TException {
return delegate.readBinary();
}

public void reset() {
delegate.reset();
}

public String toString() {
return delegate.toString();
}

}
13 changes: 9 additions & 4 deletions src/main/java/parquet/format/Util.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TIOStreamTransport;

/**
Expand Down Expand Up @@ -34,12 +35,16 @@ public static FileMetaData readFileMetaData(InputStream from) throws IOException
return read(from, new FileMetaData());
}

private static TCompactProtocol protocol(OutputStream to) {
return new TCompactProtocol(new TIOStreamTransport(to));
private static TProtocol protocol(OutputStream to) {
return protocol(new TIOStreamTransport(to));
}

private static TCompactProtocol protocol(InputStream from) {
return new TCompactProtocol(new TIOStreamTransport(from));
private static TProtocol protocol(InputStream from) {
return protocol(new TIOStreamTransport(from));
}

private static InterningProtocol protocol(TIOStreamTransport t) {
return new InterningProtocol(new TCompactProtocol(t));
}

private static <T extends TBase<?,?>> T read(InputStream from, T tbase) throws IOException {
Expand Down