Add support for Config setting in ValuesWriter factory

Author: Piyush Narang

parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java

@@ -305,8 +305,8 @@ public void testFloat_V2_NoDict() {
 
   private void doTestValueWriter(PrimitiveTypeName typeName, WriterVersion version, boolean enableDictionary, Class<? extends ValuesWriter> expectedValueWriterClass) {
     ColumnDescriptor mockPath = getMockColumn(typeName);
-    ValuesWriterFactory selectionStrategy = getDefaultFactory(version, enableDictionary);
-    ValuesWriter writer = selectionStrategy.newValuesWriter(mockPath);
+    ValuesWriterFactory factory = getDefaultFactory(version, enableDictionary);
+    ValuesWriter writer = factory.newValuesWriter(mockPath);
 
     validateWriterType(writer, expectedValueWriterClass);
   }

parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.factory;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Used with {@link ValuesWriterFactory} to indicate a factory that has Hadoop config included.
+ */
+public interface ConfigurableFactory {
+
+  void setConfiguration(Configuration config);
+
+  Configuration getConfiguration();
+}

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java

@@ -38,7 +38,7 @@
 import org.apache.parquet.Log;
 import org.apache.parquet.column.ParquetProperties;
 import org.apache.parquet.column.ParquetProperties.WriterVersion;
-import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory;
+import org.apache.parquet.column.values.factory.ConfigurableFactory;
 import org.apache.parquet.column.values.factory.ValuesWriterFactory;
 import org.apache.parquet.hadoop.ParquetFileWriter.Mode;
 import org.apache.parquet.hadoop.api.WriteSupport;
@@ -340,7 +340,13 @@ public static ValuesWriterFactory getValuesWriterFactory(Configuration conf) {
     }
 
     try {
-      return factoryOverride.newInstance();
+      ValuesWriterFactory factory = factoryOverride.newInstance();
+      if (factory instanceof ConfigurableFactory) {
+        ConfigurableFactory configurableFactory = (ConfigurableFactory) factory;
+        configurableFactory.setConfiguration(conf);
+      }
+
+      return factory;
     } catch (Exception e) {
       throw new BadConfigurationException("Unable to instantiate ValuesWriterFactory: " + factoryOverride, e);
     }

parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.column.values.factory.ConfigurableFactory;
+import org.apache.parquet.column.values.factory.ValuesWriterFactory;
+import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams;
+
+/**
+ * ValuesWriter factory test class for verifying overrides and configuration
+ */
+public class StubConfigurableValuesWriterFactory implements ValuesWriterFactory, ConfigurableFactory {
+
+  private Configuration config;
+
+  @Override
+  public void setConfiguration(Configuration config) {
+    this.config = config;
+  }
+
+  @Override
+  public Configuration getConfiguration() {
+    return config;
+  }
+
+  @Override
+  public void initialize(ValuesWriterFactoryParams params) {
+  }
+
+  @Override
+  public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) {
+    return null;
+  }
+}

parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java

@@ -19,11 +19,13 @@
 package org.apache.parquet.hadoop;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.column.values.factory.ConfigurableFactory;
 import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory;
 import org.apache.parquet.column.values.factory.ValuesWriterFactory;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
 public class TestParquetOutputFormatFactoryOverrides {
 
@@ -59,4 +61,18 @@ public void testValidFactoryOverride() {
     ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf);
     assertEquals("Incorrect factory override chosen", StubValuesWriterFactory.class, factory.getClass());
   }
+
+  @Test
+  public void testFactoryOverrideWithCfg() {
+    Configuration conf = new Configuration();
+    conf.set("foo", "bar");
+    conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.hadoop.StubConfigurableValuesWriterFactory");
+
+    ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf);
+    assertEquals("Incorrect factory override chosen", StubConfigurableValuesWriterFactory.class, factory.getClass());
+
+    ConfigurableFactory configurable = (ConfigurableFactory) factory;
+    assertNotNull("Not a ConfigurableFactory", configurable);
+    assertEquals("Incorrect config value found", "bar", configurable.getConfiguration().get("foo"));
+  }
 }