Add reader and writer for Puffin, indexes and stats file format

build.gradle

@@ -220,6 +220,7 @@ project(':iceberg-core') {
       exclude group: 'org.tukaani' // xz compression is not supported
     }
 
+    implementation 'io.airlift:aircompressor'
     implementation 'org.apache.httpcomponents.client5:httpclient5'
     implementation "com.fasterxml.jackson.core:jackson-databind"
     implementation "com.fasterxml.jackson.core:jackson-core"

bundled-guava/src/main/java/org/apache/iceberg/GuavaClasses.java

@@ -49,6 +49,7 @@
 import com.google.common.hash.Hashing;
 import com.google.common.io.CountingOutputStream;
 import com.google.common.io.Files;
+import com.google.common.io.Resources;
 import com.google.common.primitives.Bytes;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -89,6 +90,7 @@ public class GuavaClasses {
     Hashing.class.getName();
     Files.class.getName();
     Bytes.class.getName();
+    Resources.class.getName();
     MoreExecutors.class.getName();
     ThreadFactoryBuilder.class.getName();
     Iterables.class.getName();

core/src/main/java/org/apache/iceberg/io/IOUtil.java

@@ -22,12 +22,16 @@
 import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
 
 public class IOUtil {
   // not meant to be instantiated
   private IOUtil() {
   }
 
+  private static final int WRITE_CHUNK_SIZE = 8192;
+
   /**
    * Reads into a buffer from a stream, making multiple read calls if necessary.
    *
@@ -46,6 +50,21 @@ public static void readFully(InputStream stream, byte[] bytes, int offset, int l
     }
   }
 
+  /**
+   * Writes a buffer into a stream, making multiple write calls if necessary.
+   */
+  public static void writeFully(OutputStream outputStream, ByteBuffer buffer) throws IOException {
+    if (!buffer.hasRemaining()) {
+      return;
+    }
+    byte[] chunk = new byte[WRITE_CHUNK_SIZE];
+    while (buffer.hasRemaining()) {
+      int chunkSize = Math.min(chunk.length, buffer.remaining());
+      buffer.get(chunk, 0, chunkSize);
+      outputStream.write(chunk, 0, chunkSize);
+    }
+  }
+
   /**
    * Reads into a buffer from a stream, making multiple read calls if necessary
    * returning the number of bytes read until end of stream.

core/src/main/java/org/apache/iceberg/puffin/Blob.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.puffin;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+
+public final class Blob {
+  private final String type;
+  private final List<Integer> inputFields;
+  private final ByteBuffer blobData;
+  private final PuffinCompressionCodec requestedCompression;
+  private final Map<String, String> properties;
+
+  public Blob(String type, List<Integer> inputFields, ByteBuffer blobData) {
+    this(type, inputFields, blobData, null, ImmutableMap.of());
+  }
+
+  public Blob(
+      String type, List<Integer> inputFields, ByteBuffer blobData,
+      @Nullable PuffinCompressionCodec requestedCompression, Map<String, String> properties) {
+    Preconditions.checkNotNull(type, "type is null");
+    Preconditions.checkNotNull(inputFields, "inputFields is null");
+    Preconditions.checkNotNull(blobData, "blobData is null");
+    Preconditions.checkNotNull(properties, "properties is null");
+    this.type = type;
+    this.inputFields = ImmutableList.copyOf(inputFields);
+    this.blobData = blobData;
+    this.requestedCompression = requestedCompression;
+    this.properties = ImmutableMap.copyOf(properties);
+  }
+
+  public String type() {
+    return type;
+  }
+
+  public List<Integer> inputFields() {
+    return inputFields;
+  }
+
+  public ByteBuffer blobData() {
+    return blobData;
+  }
+
+  @Nullable
+  public PuffinCompressionCodec requestedCompression() {
+    return requestedCompression;
+  }
+
+  public Map<String, String> properties() {
+    return properties;
+  }
+}

core/src/main/java/org/apache/iceberg/puffin/BlobMetadata.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.puffin;
+
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+
+public class BlobMetadata {
+  private final String type;
+  private final List<Integer> inputFields;
+  private final long offset;
+  private final long length;
+  private final String compressionCodec;
+  private final Map<String, String> properties;
+
+  public BlobMetadata(
+      String type, List<Integer> inputFields, long offset, long length,
+      @Nullable String compressionCodec, Map<String, String> properties) {
+    Preconditions.checkNotNull(type, "type is null");
+    Preconditions.checkNotNull(inputFields, "inputFields is null");
+    Preconditions.checkNotNull(properties, "properties is null");
+    this.type = type;
+    this.inputFields = ImmutableList.copyOf(inputFields);
+    this.offset = offset;
+    this.length = length;
+    this.compressionCodec = compressionCodec;
+    this.properties = ImmutableMap.copyOf(properties);
+  }
+
+  public String type() {
+    return type;
+  }
+
+  public List<Integer> inputFields() {
+    return inputFields;
+  }
+
+  /**
+   * Offset in the file
+   */
+  public long offset() {
+    return offset;
+  }
+
+  /**
+   * Length in the file
+   */
+  public long length() {
+    return length;
+  }
+
+  @Nullable
+  public String compressionCodec() {
+    return compressionCodec;
+  }
+
+  public Map<String, String> properties() {
+    return properties;
+  }
+}

core/src/main/java/org/apache/iceberg/puffin/FileMetadata.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.puffin;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+
+public class FileMetadata {
+  private final List<BlobMetadata> blobs;
+  private final Map<String, String> properties;
+
+  public FileMetadata(List<BlobMetadata> blobs, Map<String, String> properties) {
+    Preconditions.checkNotNull(blobs, "blobs is null");
+    Preconditions.checkNotNull(properties, "properties is null");
+    this.blobs = ImmutableList.copyOf(blobs);
+    this.properties = ImmutableMap.copyOf(properties);
+  }
+
+  public List<BlobMetadata> blobs() {
+    return blobs;
+  }
+
+  public Map<String, String> properties() {
+    return properties;
+  }
+}