Skip to content

Commit bb8c72d

Browse files
authored
PARQUET-2465: Fall back to HadoopConfig (#1339) (#1342)
Add fallback logic We see that this causes the 1.14 to be incompatible with the previous releases. The config will be created and right after that the `getWriteSupport(conf)` is called. But since this method is freshly introduced: ```java protected WriteSupport<T> getWriteSupport(ParquetConfiguration conf) { throw new UnsupportedOperationException( "Override ParquetWriter$Builder#getWriteSupport(ParquetConfiguration)"); } ```
1 parent 0d43773 commit bb8c72d

File tree

4 files changed

+16
-7
lines changed

4 files changed

+16
-7
lines changed

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -495,16 +495,17 @@ protected Builder(OutputFile path) {
495495
/**
496496
* @param conf a configuration
497497
* @return an appropriate WriteSupport for the object model.
498+
* @deprecated Use {@link #getWriteSupport(ParquetConfiguration)} instead
498499
*/
500+
@Deprecated
499501
protected abstract WriteSupport<T> getWriteSupport(Configuration conf);
500502

501503
/**
502504
* @param conf a configuration
503505
* @return an appropriate WriteSupport for the object model.
504506
*/
505507
protected WriteSupport<T> getWriteSupport(ParquetConfiguration conf) {
506-
throw new UnsupportedOperationException(
507-
"Override ParquetWriter$Builder#getWriteSupport(ParquetConfiguration)");
508+
return getWriteSupport(ConfigurationUtil.createHadoopConfiguration(conf));
508509
}
509510

510511
/**

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/api/InitContext.java

+2
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ public Map<String, String> getMergedKeyValueMetaData() {
7979

8080
/**
8181
* @return the configuration for this job
82+
* @deprecated Use {@link #getParquetConfiguration()} instead
8283
*/
84+
@Deprecated
8385
public Configuration getConfiguration() {
8486
return ConfigurationUtil.createHadoopConfiguration(configuration);
8587
}

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/api/ReadSupport.java

+7-4
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.Map;
2222
import org.apache.hadoop.conf.Configuration;
2323
import org.apache.parquet.conf.ParquetConfiguration;
24+
import org.apache.parquet.hadoop.util.ConfigurationUtil;
2425
import org.apache.parquet.io.api.RecordMaterializer;
2526
import org.apache.parquet.schema.MessageType;
2627
import org.apache.parquet.schema.MessageTypeParser;
@@ -76,12 +77,12 @@ public ReadContext init(Configuration configuration, Map<String, String> keyValu
7677
* @param keyValueMetaData the app specific metadata from the file
7778
* @param fileSchema the schema of the file
7879
* @return the readContext that defines how to read the file
79-
* @deprecated override {@link ReadSupport#init(InitContext)} instead
80+
* @deprecated override {@link #init(InitContext)} instead
8081
*/
8182
@Deprecated
8283
public ReadContext init(
8384
ParquetConfiguration configuration, Map<String, String> keyValueMetaData, MessageType fileSchema) {
84-
throw new UnsupportedOperationException("Override ReadSupport.init(InitContext)");
85+
return init(ConfigurationUtil.createHadoopConfiguration(configuration), keyValueMetaData, fileSchema);
8586
}
8687

8788
/**
@@ -103,7 +104,9 @@ public ReadContext init(InitContext context) {
103104
* @param fileSchema the schema of the file
104105
* @param readContext returned by the init method
105106
* @return the recordMaterializer that will materialize the records
107+
* @deprecated override {@link #prepareForRead(ParquetConfiguration,Map,MessageType,ReadContext)} instead
106108
*/
109+
@Deprecated
107110
public abstract RecordMaterializer<T> prepareForRead(
108111
Configuration configuration,
109112
Map<String, String> keyValueMetaData,
@@ -125,8 +128,8 @@ public RecordMaterializer<T> prepareForRead(
125128
Map<String, String> keyValueMetaData,
126129
MessageType fileSchema,
127130
ReadContext readContext) {
128-
throw new UnsupportedOperationException(
129-
"Override ReadSupport.prepareForRead(ParquetConfiguration, Map<String, String>, MessageType, ReadContext)");
131+
return prepareForRead(
132+
ConfigurationUtil.createHadoopConfiguration(configuration), keyValueMetaData, fileSchema, readContext);
130133
}
131134

132135
/**

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/api/WriteSupport.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.Objects;
2525
import org.apache.hadoop.conf.Configuration;
2626
import org.apache.parquet.conf.ParquetConfiguration;
27+
import org.apache.parquet.hadoop.util.ConfigurationUtil;
2728
import org.apache.parquet.io.api.RecordConsumer;
2829
import org.apache.parquet.schema.MessageType;
2930

@@ -99,7 +100,9 @@ public Map<String, String> getExtraMetaData() {
99100
*
100101
* @param configuration the job's configuration
101102
* @return the information needed to write the file
103+
* @deprecated override {@link #init(ParquetConfiguration)} instead
102104
*/
105+
@Deprecated
103106
public abstract WriteContext init(Configuration configuration);
104107

105108
/**
@@ -109,7 +112,7 @@ public Map<String, String> getExtraMetaData() {
109112
* @return the information needed to write the file
110113
*/
111114
public WriteContext init(ParquetConfiguration configuration) {
112-
throw new UnsupportedOperationException("Override WriteSupport#init(ParquetConfiguration)");
115+
return init(ConfigurationUtil.createHadoopConfiguration(configuration));
113116
}
114117

115118
/**

0 commit comments

Comments
 (0)