Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public class ThriftReadSupport<T> extends ReadSupport<T> {
* Accepts a ";" delimited list of globs in the syntax implemented by {@link DeprecatedFieldProjectionFilter}
*/
@Deprecated
public static final String DEPRECATED_THRIFT_COLUMN_FILTER_KEY = "parquet.thrift.column.filter";
public static final String THRIFT_COLUMN_FILTER_KEY = "parquet.thrift.column.filter";

/**
* Accepts a ";" delimited list of glob paths, in the syntax implemented by {@link StrictFieldProjectionFilter}
Expand Down Expand Up @@ -99,15 +99,15 @@ public static void setRecordConverterClass(Configuration conf,

@Deprecated
public static void setProjectionPushdown(JobConf jobConf, String projectionString) {
jobConf.set(DEPRECATED_THRIFT_COLUMN_FILTER_KEY, projectionString);
jobConf.set(THRIFT_COLUMN_FILTER_KEY, projectionString);
}

public static void setStrictFieldProjectionFilter(Configuration conf, String semicolonDelimitedGlobs) {
conf.set(STRICT_THRIFT_COLUMN_FILTER_KEY, semicolonDelimitedGlobs);
}

public static FieldProjectionFilter getFieldProjectionFilter(Configuration conf) {
String deprecated = conf.get(DEPRECATED_THRIFT_COLUMN_FILTER_KEY);
String deprecated = conf.get(THRIFT_COLUMN_FILTER_KEY);
String strict = conf.get(STRICT_THRIFT_COLUMN_FILTER_KEY);

if (Strings.isNullOrEmpty(deprecated) && Strings.isNullOrEmpty(strict)) {
Expand All @@ -117,18 +117,18 @@ public static FieldProjectionFilter getFieldProjectionFilter(Configuration conf)
if(!Strings.isNullOrEmpty(deprecated) && !Strings.isNullOrEmpty(strict)) {
throw new ThriftProjectionException(
"You cannot provide both "
+ DEPRECATED_THRIFT_COLUMN_FILTER_KEY
+ THRIFT_COLUMN_FILTER_KEY
+ " and "
+ STRICT_THRIFT_COLUMN_FILTER_KEY
+"! "
+ DEPRECATED_THRIFT_COLUMN_FILTER_KEY
+ THRIFT_COLUMN_FILTER_KEY
+ " is deprecated."
);
}

if (!Strings.isNullOrEmpty(deprecated)) {
LOG.warn(String.format("Using %s is deprecated. Please see the docs for %s!",
DEPRECATED_THRIFT_COLUMN_FILTER_KEY, STRICT_THRIFT_COLUMN_FILTER_KEY));
THRIFT_COLUMN_FILTER_KEY, STRICT_THRIFT_COLUMN_FILTER_KEY));
return new DeprecatedFieldProjectionFilter(deprecated);
}

Expand Down Expand Up @@ -162,7 +162,7 @@ public org.apache.parquet.hadoop.api.ReadSupport.ReadContext init(InitContext co
throw new ThriftProjectionException(
String.format("You cannot provide both a partial schema and field projection filter."
+ "Only one of (%s, %s, %s) should be set.",
PARQUET_READ_SCHEMA, STRICT_THRIFT_COLUMN_FILTER_KEY, DEPRECATED_THRIFT_COLUMN_FILTER_KEY));
PARQUET_READ_SCHEMA, STRICT_THRIFT_COLUMN_FILTER_KEY, THRIFT_COLUMN_FILTER_KEY));
}

//set requestedProjections only when it's specified
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@

/**
* Filter thrift attributes using glob syntax.
* This is used for parsing values assigned to ThriftReadSupport.DEPRECATED_THRIFT_COLUMN_FILTER_KEY
* This is used for parsing values assigned to ThriftReadSupport.THRIFT_COLUMN_FILTER_KEY
* @author Tianshuo Deng
*/
@Deprecated
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
* Enhanced version of GlobPattern class that is defined in hadoop with extra capability of matching
* full path separated by '/', and double star matching
*
* This is used for parsing values assigned to ThriftReadSupport.DEPRECATED_THRIFT_COLUMN_FILTER_KEY
* This is used for parsing values assigned to ThriftReadSupport.THRIFT_COLUMN_FILTER_KEY
*
* @author Tianshuo Deng
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ public void testPullInPrimitiveValues() throws Exception {

private void shouldDoProjectionWithThriftColumnFilter(String filterDesc, TBase toWrite, TBase toRead, Class<? extends TBase<?, ?>> thriftClass) throws Exception {
Configuration conf = new Configuration();
conf.set(ThriftReadSupport.DEPRECATED_THRIFT_COLUMN_FILTER_KEY, filterDesc);
conf.set(ThriftReadSupport.THRIFT_COLUMN_FILTER_KEY, filterDesc);
shouldDoProjection(conf, toWrite, toRead, thriftClass);
}

Expand Down
1 change: 0 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,6 @@
<dumpDetails>true</dumpDetails>
<previousVersion>${previous.version}</previousVersion>
<excludes>
<exclude>org/apache/parquet/hadoop/thrift/**</exclude>
<exclude>org/apache/parquet/thrift/projection/**</exclude>
<exclude>org/apache/parquet/thrift/ThriftSchemaConverter</exclude>
<exclude>org/apache/parquet/filter2/**</exclude>
Expand Down