PARQUET-2468: ParquetMetadata must convert to json (#1349) (#1360)

wgtmac · RustedBones · web-flow · commit 9c8fde0c7a8b · 2024-05-31T09:39:23.000+08:00
Co-authored-by: Michel Davit &lt;michel@davit.fr&gt;
diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml
@@ -118,11 +118,21 @@
       <artifactId>jackson-core</artifactId>
       <version>${jackson.version}</version>
     </dependency>
+    <dependency>
+      <groupId>${jackson.groupId}</groupId>
+      <artifactId>jackson-annotations</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
     <dependency>
       <groupId>${jackson.groupId}</groupId>
       <artifactId>jackson-databind</artifactId>
       <version>${jackson-databind.version}</version>
     </dependency>
+    <dependency>
+      <groupId>${jackson.datatype.groupId}</groupId>
+      <artifactId>jackson-datatype-jdk8</artifactId>
+      <version>${jackson-modules-java8.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.xerial.snappy</groupId>
       <artifactId>snappy-java</artifactId>
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java
@@ -22,6 +22,7 @@
 import static org.apache.parquet.column.Encoding.RLE_DICTIONARY;
 import static org.apache.parquet.format.Util.readColumnMetaData;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.Set;
@@ -338,6 +339,7 @@ public ColumnPath getPath() {
    * @deprecated will be removed in 2.0.0. Use {@link #getPrimitiveType()} instead.
    */
   @Deprecated
+  @JsonIgnore
   public PrimitiveTypeName getType() {
     decryptIfNeeded();
     return properties.getType();
@@ -380,13 +382,15 @@ public PrimitiveType getPrimitiveType() {
   /**
    * @return the stats for this column
    */
+  @JsonIgnore
   public abstract Statistics getStatistics();
 
   /**
    * Method should be considered private
    *
    * @return the size stats for this column
    */
+  @JsonIgnore
   public SizeStatistics getSizeStatistics() {
     throw new UnsupportedOperationException("SizeStatistics is not implemented");
   }
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkProperties.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkProperties.java
@@ -18,6 +18,7 @@
  */
 package org.apache.parquet.hadoop.metadata;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.util.Arrays;
 import java.util.Set;
 import org.apache.parquet.column.Encoding;
@@ -76,6 +77,7 @@ public ColumnPath getPath() {
    * @deprecated will be removed in 2.0.0. Use {@link #getPrimitiveType()} instead.
    */
   @Deprecated
+  @JsonIgnore
   public PrimitiveTypeName getType() {
     return type.getPrimitiveTypeName();
   }
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/FileMetaData.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/FileMetaData.java
@@ -20,6 +20,7 @@
 
 import static java.util.Collections.unmodifiableMap;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.io.Serializable;
 import java.util.Map;
 import java.util.Objects;
@@ -109,6 +110,7 @@ public String getCreatedBy() {
     return createdBy;
   }
 
+  @JsonIgnore
   public InternalFileDecryptor getFileDecryptor() {
     return fileDecryptor;
   }
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java
@@ -19,6 +19,9 @@
 package org.apache.parquet.hadoop.metadata;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.StringWriter;
@@ -32,6 +35,14 @@ public class ParquetMetadata {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  static {
+    // Enable FAIL_ON_EMPTY_BEANS on objectmapper. Without this feature parquet-casdacing tests fail,
+    // because LogicalTypeAnnotation implementations are classes without any property.
+    objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS);
+    // Add support for Java 8 Optional
+    objectMapper.registerModule(new Jdk8Module());
+  }
+
   /**
    * @param parquetMetaData an instance of parquet metadata to convert
    * @return the json representation
@@ -50,19 +61,23 @@ public static String toPrettyJSON(ParquetMetadata parquetMetaData) {
 
   private static String toJSON(ParquetMetadata parquetMetaData, boolean isPrettyPrint) {
     try (StringWriter stringWriter = new StringWriter()) {
+      Object objectToPrint;
+      if (parquetMetaData.getFileMetaData() == null
+          || parquetMetaData.getFileMetaData().getEncryptionType()
+              == FileMetaData.EncryptionType.UNENCRYPTED) {
+        objectToPrint = parquetMetaData;
+      } else {
+        objectToPrint = parquetMetaData.getFileMetaData();
+      }
+
+      ObjectWriter writer;
       if (isPrettyPrint) {
-        Object objectToPrint;
-        if (parquetMetaData.getFileMetaData() == null
-            || parquetMetaData.getFileMetaData().getEncryptionType()
-                == FileMetaData.EncryptionType.UNENCRYPTED) {
-          objectToPrint = parquetMetaData;
-        } else {
-          objectToPrint = parquetMetaData.getFileMetaData();
-        }
-        objectMapper.writerWithDefaultPrettyPrinter().writeValue(stringWriter, objectToPrint);
+        writer = objectMapper.writerWithDefaultPrettyPrinter();
       } else {
-        objectMapper.writeValue(stringWriter, parquetMetaData);
+        writer = objectMapper.writer();
       }
+
+      writer.writeValue(stringWriter, objectToPrint);
       return stringWriter.toString();
     } catch (IOException e) {
       throw new RuntimeException(e);
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -87,6 +87,10 @@
 import org.apache.parquet.column.statistics.LongStatistics;
 import org.apache.parquet.column.statistics.SizeStatistics;
 import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.crypto.DecryptionPropertiesFactory;
+import org.apache.parquet.crypto.EncryptionPropertiesFactory;
+import org.apache.parquet.crypto.FileDecryptionProperties;
+import org.apache.parquet.crypto.InternalFileDecryptor;
 import org.apache.parquet.example.Paper;
 import org.apache.parquet.example.data.Group;
 import org.apache.parquet.example.data.simple.SimpleGroup;
@@ -635,18 +639,49 @@ public void randomTestFilterMetaData() {
   }
 
   @Test
-  public void testNullFieldMetadataDebugLogging() {
+  public void testFieldMetadataDebugLogging() {
     MessageType schema = parseMessageType("message test { optional binary some_null_field; }");
     org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData =
-        new org.apache.parquet.hadoop.metadata.FileMetaData(schema, new HashMap<String, String>(), null);
-    List<BlockMetaData> blockMetaDataList = new ArrayList<BlockMetaData>();
+        new org.apache.parquet.hadoop.metadata.FileMetaData(
+            schema,
+            new HashMap<>(),
+            null,
+            org.apache.parquet.hadoop.metadata.FileMetaData.EncryptionType.UNENCRYPTED,
+            null);
+    List<BlockMetaData> blockMetaDataList = new ArrayList<>();
     BlockMetaData blockMetaData = new BlockMetaData();
     blockMetaData.addColumn(createColumnChunkMetaData());
     blockMetaDataList.add(blockMetaData);
     ParquetMetadata metadata = new ParquetMetadata(fileMetaData, blockMetaDataList);
     ParquetMetadata.toJSON(metadata);
   }
 
+  @Test
+  public void testEncryptedFieldMetadataDebugLogging() {
+    Configuration conf = new Configuration();
+    conf.set(
+        EncryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME,
+        "org.apache.parquet.crypto.SampleDecryptionPropertiesFactory");
+    DecryptionPropertiesFactory decryptionPropertiesFactory = DecryptionPropertiesFactory.loadFactory(conf);
+    FileDecryptionProperties decryptionProperties =
+        decryptionPropertiesFactory.getFileDecryptionProperties(conf, null);
+
+    MessageType schema = parseMessageType("message test { optional binary some_null_field; }");
+
+    org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData =
+        new org.apache.parquet.hadoop.metadata.FileMetaData(
+            schema,
+            new HashMap<>(),
+            null,
+            org.apache.parquet.hadoop.metadata.FileMetaData.EncryptionType.ENCRYPTED_FOOTER,
+            new InternalFileDecryptor(decryptionProperties));
+
+    List<BlockMetaData> blockMetaDataList = new ArrayList<>();
+    ParquetMetadata metadata = new ParquetMetadata(fileMetaData, blockMetaDataList);
+    ParquetMetadata.toJSON(metadata);
+    System.out.println(ParquetMetadata.toPrettyJSON(metadata));
+  }
+
   @Test
   public void testMetadataToJson() {
     ParquetMetadata metadata = new ParquetMetadata(null, null);
diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml
@@ -38,11 +38,22 @@
       <artifactId>jackson-core</artifactId>
       <version>${jackson.version}</version>
     </dependency>
+    <dependency>
+      <groupId>${jackson.groupId}</groupId>
+      <artifactId>jackson-annotations</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
       <version>${jackson-databind.version}</version>
     </dependency>
+    <!-- Add support for Java 8 Optional -->
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-jdk8</artifactId>
+      <version>${jackson-modules-java8.version}</version>
+    </dependency>
   </dependencies>
 
   <properties>
@@ -70,6 +81,7 @@
               <artifactSet>
                 <includes>
                   <include>${jackson.groupId}:*</include>
+                  <include>${jackson.datatype.groupId}:*</include>
                 </includes>
               </artifactSet>
               <filters>
@@ -79,6 +91,12 @@
                     <include>**</include>
                   </includes>
                 </filter>
+                <filter>
+                  <artifact>${jackson.datatype.groupId}:*</artifact>
+                  <includes>
+                    <include>**</include>
+                  </includes>
+                </filter>
               </filters>
               <relocations>
                 <relocation>
diff --git a/pom.xml b/pom.xml
@@ -68,6 +68,7 @@
     <jackson.package>com.fasterxml.jackson</jackson.package>
     <jackson.version>2.17.0</jackson.version>
     <jackson-databind.version>2.17.0</jackson-databind.version>
+    <jackson-modules-java8.version>2.17.0</jackson-modules-java8.version>
     <japicmp.version>0.21.0</japicmp.version>
     <javax.annotation.version>1.3.2</javax.annotation.version>
     <spotless.version>2.30.0</spotless.version>

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@`
`20`	`20`
`21`	`21`	`import static java.util.Collections.unmodifiableMap;`
`22`	`22`
	`23`	`+import com.fasterxml.jackson.annotation.JsonIgnore;`
`23`	`24`	`import java.io.Serializable;`
`24`	`25`	`import java.util.Map;`
`25`	`26`	`import java.util.Objects;`
`@@ -109,6 +110,7 @@ public String getCreatedBy() {`
`109`	`110`	`return createdBy;`
`110`	`111`	`}`
`111`	`112`
	`113`	`+ @JsonIgnore`
`112`	`114`	`public InternalFileDecryptor getFileDecryptor() {`
`113`	`115`	`return fileDecryptor;`
`114`	`116`	`}`