diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index 420d97e280..e8b6d7384e 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -72,7 +72,6 @@ // TODO: This file has become too long! // TODO: Lets split it up: https://issues.apache.org/jira/browse/PARQUET-310 public class ParquetMetadataConverter { - private ParquetMetadataConverter() { } public static final MetadataFilter NO_FILTER = new NoFilter(); public static final MetadataFilter SKIP_ROW_GROUPS = new SkipMetadataFilter(); @@ -87,7 +86,7 @@ private ParquetMetadataConverter() { } private static final ConcurrentHashMap, Set> cachedEncodingSets = new ConcurrentHashMap, Set>(); - public static FileMetaData toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata) { + public FileMetaData toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata) { List blocks = parquetMetadata.getBlocks(); List rowGroups = new ArrayList(); int numRows = 0; @@ -111,13 +110,13 @@ public static FileMetaData toParquetMetadata(int currentVersion, ParquetMetadata } // Visible for testing - static List toParquetSchema(MessageType schema) { + List toParquetSchema(MessageType schema) { List result = new ArrayList(); addToList(result, schema); return result; } - private static void addToList(final List result, org.apache.parquet.schema.Type field) { + private void addToList(final List result, org.apache.parquet.schema.Type field) { field.accept(new TypeVisitor() { @Override public void visit(PrimitiveType primitiveType) { @@ -164,7 +163,7 @@ private void visitChildren(final List result, }); } - private static void addRowGroup(ParquetMetadata parquetMetadata, List rowGroups, BlockMetaData block) { + private void addRowGroup(ParquetMetadata parquetMetadata, List rowGroups, BlockMetaData block) { //rowGroup.total_byte_size = ; List columns = block.getColumns(); List parquetColumns = new ArrayList(); @@ -193,7 +192,7 @@ private static void addRowGroup(ParquetMetadata parquetMetadata, List rowGroups.add(rowGroup); } - private static List toFormatEncodings(Set encodings) { + private List toFormatEncodings(Set encodings) { List converted = new ArrayList(encodings.size()); for (org.apache.parquet.column.Encoding encoding : encodings) { converted.add(getEncoding(encoding)); @@ -202,7 +201,7 @@ private static List toFormatEncodings(Set fromFormatEncodings(List encodings) { + Set fromFormatEncodings(List encodings) { Set converted = new HashSet(); for (Encoding encoding : encodings) { @@ -225,11 +224,11 @@ static Set fromFormatEncodings(List() { @Override public FileMetaData visit(NoFilter filter) throws IOException { @@ -547,7 +546,7 @@ public FileMetaData visit(RangeMetadataFilter filter) throws IOException { return parquetMetadata; } - public static ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata) throws IOException { + public ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata) throws IOException { MessageType messageType = fromParquetSchema(parquetMetadata.getSchema()); List blocks = new ArrayList(); List row_groups = parquetMetadata.getRow_groups(); @@ -606,7 +605,7 @@ private static ColumnPath getPath(ColumnMetaData metaData) { } // Visible for testing - static MessageType fromParquetSchema(List schema) { + MessageType fromParquetSchema(List schema) { Iterator iterator = schema.iterator(); SchemaElement root = iterator.next(); Types.MessageTypeBuilder builder = Types.buildMessage(); @@ -614,7 +613,7 @@ static MessageType fromParquetSchema(List schema) { return builder.named(root.name); } - private static void buildChildren(Types.GroupBuilder builder, + private void buildChildren(Types.GroupBuilder builder, Iterator schema, int childrenCount) { for (int i = 0; i < childrenCount; i++) { @@ -654,17 +653,17 @@ private static void buildChildren(Types.GroupBuilder builder, } // Visible for testing - static FieldRepetitionType toParquetRepetition(Repetition repetition) { + FieldRepetitionType toParquetRepetition(Repetition repetition) { return FieldRepetitionType.valueOf(repetition.name()); } // Visible for testing - static Repetition fromParquetRepetition(FieldRepetitionType repetition) { + Repetition fromParquetRepetition(FieldRepetitionType repetition) { return Repetition.valueOf(repetition.name()); } @Deprecated - public static void writeDataPageHeader( + public void writeDataPageHeader( int uncompressedSize, int compressedSize, int valueCount, @@ -681,7 +680,7 @@ public static void writeDataPageHeader( valuesEncoding), to); } - public static void writeDataPageHeader( + public void writeDataPageHeader( int uncompressedSize, int compressedSize, int valueCount, @@ -696,7 +695,7 @@ public static void writeDataPageHeader( to); } - private static PageHeader newDataPageHeader( + private PageHeader newDataPageHeader( int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.statistics.Statistics statistics, @@ -717,7 +716,7 @@ private static PageHeader newDataPageHeader( return pageHeader; } - public static void writeDataPageV2Header( + public void writeDataPageV2Header( int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.statistics.Statistics statistics, @@ -733,7 +732,7 @@ public static void writeDataPageV2Header( rlByteLength, dlByteLength), to); } - private static PageHeader newDataPageV2Header( + private PageHeader newDataPageV2Header( int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.statistics.Statistics statistics, @@ -753,7 +752,7 @@ private static PageHeader newDataPageV2Header( return pageHeader; } - public static void writeDictionaryPageHeader( + public void writeDictionaryPageHeader( int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to) throws IOException { PageHeader pageHeader = new PageHeader(PageType.DICTIONARY_PAGE, uncompressedSize, compressedSize); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java index c90ba8a9be..0a0b316065 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java @@ -46,6 +46,8 @@ class ColumnChunkPageWriteStore implements PageWriteStore { private static final Log LOG = Log.getLog(ColumnChunkPageWriteStore.class); + private static ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); + private static final class ColumnChunkPageWriter implements PageWriter { private final ColumnDescriptor path; @@ -92,7 +94,7 @@ public void writePage(BytesInput bytes, + compressedSize); } tempOutputStream.reset(); - ParquetMetadataConverter.writeDataPageHeader( + parquetMetadataConverter.writeDataPageHeader( (int)uncompressedSize, (int)compressedSize, valueCount, @@ -131,7 +133,7 @@ public void writePageV2( compressedData.size() + repetitionLevels.size() + definitionLevels.size() ); tempOutputStream.reset(); - ParquetMetadataConverter.writeDataPageV2Header( + parquetMetadataConverter.writeDataPageV2Header( uncompressedSize, compressedSize, valueCount, nullCount, rowCount, statistics, diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 19370cd895..ea7a6723fa 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -89,7 +89,9 @@ public class ParquetFileReader implements Closeable { private static final Log LOG = Log.getLog(ParquetFileReader.class); - public static final String PARQUET_READ_PARALLELISM = "parquet.metadata.read.parallelism"; + public static String PARQUET_READ_PARALLELISM = "parquet.metadata.read.parallelism"; + + private static ParquetMetadataConverter converter = new ParquetMetadataConverter(); /** * for files provided, check if there's a summary file. @@ -426,7 +428,7 @@ public static final ParquetMetadata readFooter(Configuration configuration, File throw new RuntimeException("corrupted file: the footer index is not within the file"); } f.seek(footerIndex); - return ParquetMetadataConverter.readParquetMetadata(f, filter); + return converter.readParquetMetadata(f, filter); } finally { f.close(); } @@ -573,7 +575,7 @@ public ColumnChunkPageReader readAllPages() throws IOException { this.readAsBytesInput(compressedPageSize), uncompressedPageSize, dicHeader.getNum_values(), - ParquetMetadataConverter.getEncoding(dicHeader.getEncoding()) + converter.getEncoding(dicHeader.getEncoding()) ); break; case DATA_PAGE: @@ -587,9 +589,9 @@ public ColumnChunkPageReader readAllPages() throws IOException { createdBy, dataHeaderV1.getStatistics(), descriptor.col.getType()), - ParquetMetadataConverter.getEncoding(dataHeaderV1.getRepetition_level_encoding()), - ParquetMetadataConverter.getEncoding(dataHeaderV1.getDefinition_level_encoding()), - ParquetMetadataConverter.getEncoding(dataHeaderV1.getEncoding()) + converter.getEncoding(dataHeaderV1.getRepetition_level_encoding()), + converter.getEncoding(dataHeaderV1.getDefinition_level_encoding()), + converter.getEncoding(dataHeaderV1.getEncoding()) )); valuesCountReadSoFar += dataHeaderV1.getNum_values(); break; @@ -603,7 +605,7 @@ public ColumnChunkPageReader readAllPages() throws IOException { dataHeaderV2.getNum_values(), this.readAsBytesInput(dataHeaderV2.getRepetition_levels_byte_length()), this.readAsBytesInput(dataHeaderV2.getDefinition_levels_byte_length()), - ParquetMetadataConverter.getEncoding(dataHeaderV2.getEncoding()), + converter.getEncoding(dataHeaderV2.getEncoding()), this.readAsBytesInput(dataSize), uncompressedPageSize, fromParquetStatistics( diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index 1f872403ee..e2853763e0 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -67,6 +67,8 @@ public class ParquetFileWriter { private static final Log LOG = Log.getLog(ParquetFileWriter.class); + private static ParquetMetadataConverter metadataConverter = new ParquetMetadataConverter(); + public static final String PARQUET_METADATA_FILE = "_metadata"; public static final String PARQUET_COMMON_METADATA_FILE = "_common_metadata"; public static final byte[] MAGIC = "PAR1".getBytes(Charset.forName("ASCII")); @@ -302,7 +304,7 @@ public void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOExceptio currentChunkDictionaryPageOffset = out.getPos(); int uncompressedSize = dictionaryPage.getUncompressedSize(); int compressedPageSize = (int)dictionaryPage.getBytes().size(); // TODO: fix casts - ParquetMetadataConverter.writeDictionaryPageHeader( + metadataConverter.writeDictionaryPageHeader( uncompressedSize, compressedPageSize, dictionaryPage.getDictionarySize(), @@ -337,7 +339,7 @@ public void writeDataPage( long beforeHeader = out.getPos(); if (DEBUG) LOG.debug(beforeHeader + ": write data page: " + valueCount + " values"); int compressedPageSize = (int)bytes.size(); - ParquetMetadataConverter.writeDataPageHeader( + metadataConverter.writeDataPageHeader( uncompressedPageSize, compressedPageSize, valueCount, rlEncoding, @@ -374,7 +376,7 @@ public void writeDataPage( long beforeHeader = out.getPos(); if (DEBUG) LOG.debug(beforeHeader + ": write data page: " + valueCount + " values"); int compressedPageSize = (int)bytes.size(); - ParquetMetadataConverter.writeDataPageHeader( + metadataConverter.writeDataPageHeader( uncompressedPageSize, compressedPageSize, valueCount, statistics, @@ -467,7 +469,7 @@ public void end(Map extraMetaData) throws IOException { private static void serializeFooter(ParquetMetadata footer, FSDataOutputStream out) throws IOException { long footerIndex = out.getPos(); - org.apache.parquet.format.FileMetaData parquetMetadata = ParquetMetadataConverter.toParquetMetadata(CURRENT_VERSION, footer); + org.apache.parquet.format.FileMetaData parquetMetadata = metadataConverter.toParquetMetadata(CURRENT_VERSION, footer); writeFileMetaData(parquetMetadata, out); if (DEBUG) LOG.debug(out.getPos() + ": footer length = " + (out.getPos() - footerIndex)); BytesUtils.writeIntLittleEndian(out, (int)(out.getPos() - footerIndex)); diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index dd4aba9aa5..e44d140471 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -87,14 +87,16 @@ public void testPageHeader() throws IOException { @Test public void testSchemaConverter() { - List parquetSchema = ParquetMetadataConverter.toParquetSchema(Paper.schema); - MessageType schema = ParquetMetadataConverter.fromParquetSchema(parquetSchema); + ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); + List parquetSchema = parquetMetadataConverter.toParquetSchema(Paper.schema); + MessageType schema = parquetMetadataConverter.fromParquetSchema(parquetSchema); assertEquals(Paper.schema, schema); } @Test public void testSchemaConverterDecimal() { - List schemaElements = ParquetMetadataConverter.toParquetSchema( + ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); + List schemaElements = parquetMetadataConverter.toParquetSchema( Types.buildMessage() .required(PrimitiveTypeName.BINARY) .as(OriginalType.DECIMAL).precision(9).scale(2) @@ -123,29 +125,30 @@ public void testSchemaConverterDecimal() { @Test public void testEnumEquivalence() { + ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); for (org.apache.parquet.column.Encoding encoding : org.apache.parquet.column.Encoding.values()) { - assertEquals(encoding, ParquetMetadataConverter.getEncoding(ParquetMetadataConverter.getEncoding(encoding))); + assertEquals(encoding, parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding))); } for (org.apache.parquet.format.Encoding encoding : org.apache.parquet.format.Encoding.values()) { - assertEquals(encoding, ParquetMetadataConverter.getEncoding(ParquetMetadataConverter.getEncoding(encoding))); + assertEquals(encoding, parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding))); } for (Repetition repetition : Repetition.values()) { - assertEquals(repetition, ParquetMetadataConverter.fromParquetRepetition(ParquetMetadataConverter.toParquetRepetition(repetition))); + assertEquals(repetition, parquetMetadataConverter.fromParquetRepetition(parquetMetadataConverter.toParquetRepetition(repetition))); } for (FieldRepetitionType repetition : FieldRepetitionType.values()) { - assertEquals(repetition, ParquetMetadataConverter.toParquetRepetition(ParquetMetadataConverter.fromParquetRepetition(repetition))); + assertEquals(repetition, parquetMetadataConverter.toParquetRepetition(parquetMetadataConverter.fromParquetRepetition(repetition))); } for (PrimitiveTypeName primitiveTypeName : PrimitiveTypeName.values()) { - assertEquals(primitiveTypeName, ParquetMetadataConverter.getPrimitive(ParquetMetadataConverter.getType(primitiveTypeName))); + assertEquals(primitiveTypeName, parquetMetadataConverter.getPrimitive(parquetMetadataConverter.getType(primitiveTypeName))); } for (Type type : Type.values()) { - assertEquals(type, ParquetMetadataConverter.getType(ParquetMetadataConverter.getPrimitive(type))); + assertEquals(type, parquetMetadataConverter.getType(parquetMetadataConverter.getPrimitive(type))); } for (OriginalType original : OriginalType.values()) { - assertEquals(original, ParquetMetadataConverter.getOriginalType(ParquetMetadataConverter.getConvertedType(original))); + assertEquals(original, parquetMetadataConverter.getOriginalType(parquetMetadataConverter.getConvertedType(original))); } for (ConvertedType converted : ConvertedType.values()) { - assertEquals(converted, ParquetMetadataConverter.getConvertedType(ParquetMetadataConverter.getOriginalType(converted))); + assertEquals(converted, parquetMetadataConverter.getConvertedType(parquetMetadataConverter.getOriginalType(converted))); } } @@ -283,6 +286,8 @@ private ColumnChunkMetaData createColumnChunkMetaData() { @Test public void testEncodingsCache() { + ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); + List formatEncodingsCopy1 = Arrays.asList(org.apache.parquet.format.Encoding.BIT_PACKED, org.apache.parquet.format.Encoding.RLE_DICTIONARY, @@ -298,9 +303,9 @@ public void testEncodingsCache() { expected.add(org.apache.parquet.column.Encoding.RLE_DICTIONARY); expected.add(org.apache.parquet.column.Encoding.DELTA_LENGTH_BYTE_ARRAY); - Set res1 = ParquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy1); - Set res2 = ParquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy1); - Set res3 = ParquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy2); + Set res1 = parquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy1); + Set res2 = parquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy1); + Set res3 = parquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy2); // make sure they are all semantically equal assertEquals(expected, res1); diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java index 97d90e472a..2c05c301b5 100644 --- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java +++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java @@ -73,7 +73,7 @@ * * @author Tianshuo Deng */ -public class ThriftSchemaConvertVisitor implements ThriftType.TypeVisitor { +class ThriftSchemaConvertVisitor implements ThriftType.StateVisitor { private final FieldProjectionFilter fieldProjectionFilter; private final boolean doProjection; diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java index ec63d85183..e8e5617d32 100644 --- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java +++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java @@ -34,7 +34,7 @@ /** * Create a dummy events for all required fields according to thrift definition */ -class DefaultEventsVisitor implements ThriftType.TypeVisitor { +class DefaultEventsVisitor implements ThriftType.StateVisitor { List dummyEvents= new ArrayList(); @Override public Void visit(ThriftType.MapType mapType, Void v) { diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java index 95a6d276b0..46c26a5926 100644 --- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java +++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java @@ -44,7 +44,7 @@ public class CompatibilityChecker { public CompatibilityReport checkCompatibility(ThriftType.StructType oldStruct, ThriftType.StructType newStruct) { CompatibleCheckerVisitor visitor = new CompatibleCheckerVisitor(oldStruct); - newStruct.accept(visitor, null); + newStruct.accept(visitor); return visitor.getReport(); } @@ -68,7 +68,7 @@ public List getMessages() { } } -class CompatibleCheckerVisitor implements ThriftType.TypeVisitor { +class CompatibleCheckerVisitor implements ThriftType.TypeVisitor { ThriftType oldType; CompatibilityReport report = new CompatibilityReport(); @@ -81,7 +81,7 @@ public CompatibilityReport getReport() { } @Override - public Void visit(ThriftType.MapType mapType, Void v) { + public void visit(ThriftType.MapType mapType) { ThriftType.MapType currentOldType = ((ThriftType.MapType) oldType); ThriftField oldKeyField = currentOldType.getKey(); ThriftField newKeyField = mapType.getKey(); @@ -93,27 +93,24 @@ public Void visit(ThriftType.MapType mapType, Void v) { checkField(oldValueField, newValueField); oldType = currentOldType; - return null; } @Override - public Void visit(ThriftType.SetType setType, Void v) { + public void visit(ThriftType.SetType setType) { ThriftType.SetType currentOldType = ((ThriftType.SetType) oldType); ThriftField oldField = currentOldType.getValues(); ThriftField newField = setType.getValues(); checkField(oldField, newField); oldType = currentOldType; - return null; } @Override - public Void visit(ThriftType.ListType listType, Void v) { + public void visit(ThriftType.ListType listType) { ThriftType.ListType currentOldType = ((ThriftType.ListType) oldType); ThriftField oldField = currentOldType.getValues(); ThriftField newField = listType.getValues(); checkField(oldField, newField); oldType = currentOldType; - return null; } public void fail(String message) { @@ -138,7 +135,7 @@ private void checkField(ThriftField oldField, ThriftField newField) { } oldType = oldField.getType(); - newField.getType().accept(this, null); + newField.getType().accept(this); } private boolean firstIsMoreRestirctive(ThriftField.Requirement firstReq, ThriftField.Requirement secReq) { @@ -151,7 +148,7 @@ private boolean firstIsMoreRestirctive(ThriftField.Requirement firstReq, ThriftF } @Override - public Void visit(ThriftType.StructType newStruct, Void v) { + public void visit(ThriftType.StructType newStruct) { ThriftType.StructType currentOldType = ((ThriftType.StructType) oldType); short oldMaxId = 0; for (ThriftField oldField : currentOldType.getChildren()) { @@ -162,7 +159,7 @@ public Void visit(ThriftType.StructType newStruct, Void v) { ThriftField newField = newStruct.getChildById(fieldId); if (newField == null) { fail("can not find index in new Struct: " + fieldId +" in " + newStruct); - return null; + return; } checkField(oldField, newField); } @@ -176,58 +173,49 @@ public Void visit(ThriftType.StructType newStruct, Void v) { short newFieldId = newField.getFieldId(); if (newFieldId > oldMaxId) { fail("new required field " + newField.getName() + " is added"); - return null; + return; } if (newFieldId < oldMaxId && currentOldType.getChildById(newFieldId) == null) { fail("new required field " + newField.getName() + " is added"); - return null; + return; } } //restore oldType = currentOldType; - return null; } @Override - public Void visit(EnumType enumType, Void v) { - return null; + public void visit(EnumType enumType) { } @Override - public Void visit(BoolType boolType, Void v) { - return null; + public void visit(BoolType boolType) { } @Override - public Void visit(ByteType byteType, Void v) { - return null; + public void visit(ByteType byteType) { } @Override - public Void visit(DoubleType doubleType, Void v) { - return null; + public void visit(DoubleType doubleType) { } @Override - public Void visit(I16Type i16Type, Void v) { - return null; + public void visit(I16Type i16Type) { } @Override - public Void visit(I32Type i32Type, Void v) { - return null; + public void visit(I32Type i32Type) { } @Override - public Void visit(I64Type i64Type, Void v) { - return null; + public void visit(I64Type i64Type) { } @Override - public Void visit(StringType stringType, Void v) { - return null; + public void visit(StringType stringType) { } } diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java index 92d12b43d7..19c7c9fd4c 100644 --- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java +++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java @@ -98,7 +98,7 @@ public String toString() { return toJSON(); } - public interface TypeVisitor { + public interface StateVisitor { R visit(MapType mapType, S state); @@ -126,6 +126,85 @@ public interface TypeVisitor { } + /** + * @deprecated will be removed in 2.0.0; use StateVisitor instead. + */ + public interface TypeVisitor { + + void visit(MapType mapType); + + void visit(SetType setType); + + void visit(ListType listType); + + void visit(StructType structType); + + void visit(EnumType enumType); + + void visit(BoolType boolType); + + void visit(ByteType byteType); + + void visit(DoubleType doubleType); + + void visit(I16Type i16Type); + + void visit(I32Type i32Type); + + void visit(I64Type i64Type); + + void visit(StringType stringType); + + } + + /** + * @deprecated will be removed in 2.0.0. + */ + @Deprecated + public static abstract class ComplexTypeVisitor implements TypeVisitor { + + @Override + final public void visit(EnumType enumType) { + throw new IllegalArgumentException("Expected complex type"); + } + + @Override + final public void visit(BoolType boolType) { + throw new IllegalArgumentException("Expected complex type"); + } + + @Override + final public void visit(ByteType byteType) { + throw new IllegalArgumentException("Expected complex type"); + } + + @Override + final public void visit(DoubleType doubleType) { + throw new IllegalArgumentException("Expected complex type"); + } + + @Override + final public void visit(I16Type i16Type) { + throw new IllegalArgumentException("Expected complex type"); + } + + @Override + final public void visit(I32Type i32Type) { + throw new IllegalArgumentException("Expected complex type"); + } + + @Override + final public void visit(I64Type i64Type) { + throw new IllegalArgumentException("Expected complex type"); + } + + @Override + final public void visit(StringType stringType) { + throw new IllegalArgumentException("Expected complex type"); + } + + } + public static class StructType extends ThriftType { private final List children; @@ -189,10 +268,15 @@ public StructOrUnionType getStructOrUnionType() { } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -232,10 +316,15 @@ public ThriftField getValue() { } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -273,10 +362,15 @@ public ThriftField getValues() { } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -312,10 +406,15 @@ public ThriftField getValues() { } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -408,10 +507,15 @@ private void prepareEnumLookUp() { } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -440,9 +544,14 @@ public BoolType() { super(BOOL); } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } } public static class ByteType extends ThriftType { @@ -452,9 +561,14 @@ public ByteType() { super(BYTE); } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } } public static class DoubleType extends ThriftType { @@ -464,9 +578,14 @@ public DoubleType() { super(DOUBLE); } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } } public static class I16Type extends ThriftType { @@ -476,9 +595,14 @@ public I16Type() { super(I16); } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } } public static class I32Type extends ThriftType { @@ -488,9 +612,14 @@ public I32Type() { super(I32); } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } } public static class I64Type extends ThriftType { @@ -500,10 +629,15 @@ public I64Type() { super(I64); } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } } public static class StringType extends ThriftType { @@ -513,9 +647,14 @@ public StringType() { super(STRING); } @Override - public R accept(TypeVisitor visitor, S state) { + public R accept(StateVisitor visitor, S state) { return visitor.visit(this, state); } + + @Override + public void accept(TypeVisitor visitor) { + visitor.visit(this); + } } private final ThriftTypeID type; @@ -525,7 +664,9 @@ private ThriftType(ThriftTypeID type) { this.type = type; } - public abstract R accept(TypeVisitor visitor, S state); + public abstract void accept(TypeVisitor visitor); + + public abstract R accept(StateVisitor visitor, S state); @JsonIgnore public ThriftTypeID getType() { diff --git a/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java b/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java index ee545dbd1a..a00718a568 100644 --- a/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java +++ b/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java @@ -69,7 +69,7 @@ public void testFieldsPath() { } - private static class PrimitivePathVisitor implements ThriftType.TypeVisitor, FieldsPath> { + private static class PrimitivePathVisitor implements ThriftType.StateVisitor, FieldsPath> { private String delim; private PrimitivePathVisitor(String delim) { diff --git a/pom.xml b/pom.xml index c171660b3d..d48021cc8e 100644 --- a/pom.xml +++ b/pom.xml @@ -226,7 +226,10 @@ org/apache/parquet/column/** org/apache/parquet/hadoop/ParquetInputSplit shaded/** - parquet/** + + org/apache/parquet/Version + org/apache/parquet/schema/** + org/apache/parquet/thrift/ThriftSchemaConvertVisitor