PARQUET-423 Make writing Avro to Parquet less noisy

.gitignore

@@ -16,3 +16,6 @@ dependency-reduced-pom.xml
 parquet-scrooge/.cache
 .idea/*
 target/
+.cache
+*~
+mvn_install.log

.travis.yml

@@ -4,7 +4,7 @@ before_install:
   - sudo apt-get install build-essential
   - mkdir protobuf_install
   - pushd protobuf_install
-  - wget http://protobuf.googlecode.com/files/protobuf-2.5.0.tar.gz
+  - wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
   - tar xzf protobuf-2.5.0.tar.gz
   - cd  protobuf-2.5.0
   - ./configure
@@ -24,8 +24,8 @@ before_install:
   - cd ..
 
 env:
-  - HADOOP_PROFILE=default
-  - HADOOP_PROFILE=hadoop-2
+  - HADOOP_PROFILE=hadoop-1 TEST_CODECS=uncompressed
+  - HADOOP_PROFILE=default TEST_CODECS=gzip,snappy
 
 install: mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || (cat mvn_install.log && false)
 script: mvn test -P $HADOOP_PROFILE

LICENSE

@@ -178,6 +178,14 @@
 
 --------------------------------------------------------------------------------
 
+This product includes code from Apache Avro.
+
+Copyright: 2014 The Apache Software Foundation.
+Home page: https://avro.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
 This project includes code from Daniel Lemire's JavaFastPFOR project. The
 "Lemire" bit packing source code produced by parquet-generator is derived from
 the JavaFastPFOR project.

NOTICE

@@ -43,3 +43,14 @@ with the following copyright notice:
   See the License for the specific language governing permissions and
   limitations under the License.
 
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Avro, which includes the following in
+its NOTICE file:
+
+  Apache Avro
+  Copyright 2010-2015 The Apache Software Foundation
+
+  This product includes software developed at
+  The Apache Software Foundation (http://www.apache.org/).
+

README.md

@@ -35,7 +35,7 @@ Parquet-MR uses Maven to build and depends on both the thrift and protoc compile
 To build and install the protobuf compiler, run:
 
 ```
-wget http://protobuf.googlecode.com/files/protobuf-2.5.0.tar.gz
+wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
 tar xzf protobuf-2.5.0.tar.gz
 cd  protobuf-2.5.0
 ./configure
@@ -62,7 +62,7 @@ sudo make install
 Once protobuf and thrift are available in your path, you can build the project by running:
 
 ```
-mvn clean install
+LC_ALL=C mvn clean install
 ```
 
 ## Features
@@ -111,8 +111,8 @@ Avro conversion is implemented via the [parquet-avro](https://github.com/apache/
 * the ParquetInputFormat can be provided a ReadSupport to materialize your own objects by implementing a RecordMaterializer
 
 See the APIs:
-* [Record conversion API](https://github.com/apache/parquet-mr/tree/master/parquet-column/src/main/java/parquet/io/api)
-* [Hadoop API](https://github.com/apache/parquet-mr/tree/master/parquet-hadoop/src/main/java/parquet/hadoop/api)
+* [Record conversion API](https://github.com/apache/parquet-mr/tree/master/parquet-column/src/main/java/org/apache/parquet/io/api)
+* [Hadoop API](https://github.com/apache/parquet-mr/tree/master/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/api)
 
 ## Apache Pig integration
 A [Loader](https://github.com/apache/parquet-mr/blob/master/parquet-pig/src/main/java/org/apache/parquet/pig/ParquetLoader.java) and a [Storer](https://github.com/apache/parquet-mr/blob/master/parquet-pig/src/main/java/org/apache/parquet/pig/ParquetStorer.java) are provided to read and write Parquet files with Apache Pig
@@ -202,16 +202,8 @@ Thank you for getting involved!
 
 ## Authors and contributors
 
-* Julien Le Dem [@J_](http://twitter.com/J_) <https://github.com/julienledem>
-* Tom White <https://github.com/tomwhite>
-* Mickaël Lacour <https://github.com/mickaellcr>
-* Remy Pecqueur <https://github.com/Lordshinjo>
-* Avi Bryant <https://github.com/avibryant>
-* Dmitriy Ryaboy [@squarecog](https://twitter.com/squarecog) <https://github.com/dvryaboy>
-* Jonathan Coveney <http://twitter.com/jco>
-* Brock Noland <https://github.com/brockn>
-* Tianshuo Deng <https://github.com/tsdeng>
-* and many others -- see the [Contributor report]( https://github.com/apache/parquet-mr/contributors)
+* [Contributors](https://github.com/apache/parquet-mr/graphs/contributors)
+* [Committers](dev/COMMITTERS.md)
 
 ## Code of Conduct
 

dev/COMMITTERS.md

@@ -17,30 +17,41 @@
   ~ under the License.
   -->
 
-# Committers (in aplhabetical order):
+# Committers (in alphabetical order):
 
-| Name               | Apache Id  | github id      | JIRA id     |
-|--------------------|------------|----------------|-------------|
-| Aniket Mokashi     | aniket486  | aniket486      |             |
-| Brock Noland       | brock      | brockn         |             |
-| Cheng Lian         | lian       | liancheng      | lian cheng  |
-| Chris Aniszczyk    | caniszczyk |                |             |
-| Dmitriy Ryaboy     | dvryaboy   | dvryaboy       |             |
-| Jake Farrell       | jfarrell   |                |             |
-| Jonathan Coveney   | jcoveney   | jcoveney       |             |
-| Julien Le Dem      | julien     | julienledem    | julienledem |
-| Lukas Nalezenec    | lukas      | lukasnalezenec |             |
-| Marcel Kornacker   | marcel     |                |             |
-| Mickael Lacour     | mlacour    | mickaellcr     |             |
-| Nong Li            | nong       | nongli         |             |
-| Remy Pecqueur      | rpecqueur  | Lordshinjo     |             |
-| Ryan Blue          | blue       | rdblue         |             |
-| Sergio Pena        | spena      | spena          | spena       |
-| Tianshuo Deng      | tianshuo   | tsdeng         |             |
-| Tom White          | tomwhite   | tomwhite       |             |
-| Wesley Graham Peck | wesleypeck | wesleypeck     |             |
+The official list of committers can be found here: [Apache Parquet Committers and PMC](http://people.apache.org/committers-by-project.html#parquet)
 
-Reviewing guidelines:
+Below is more information about each committer (in alphabetical order). If this information becomes out of date, please send a PR to update!
+
+| Name                   | Apache Id       | github id           | JIRA id        |
+|------------------------|-----------------|---------------------|----------------|
+| Alex Levenson          | alexlevenson    | @isnotinvain        | alexlevenson   |                            
+| Aniket Mokashi         | aniket486       | @aniket486          |                |
+| Brock Noland           | brock           | @brockn             |                |
+| Cheng Lian             | lian            | @liancheng          | liancheng      |
+| Chris Aniszczyk        | caniszczyk      | @caniszczyk         |                |
+| Chris Mattmann         | mattmann        | @chrismattmann      |                |
+| Daniel C. Weeks        | dweeks          | @danielcweeks       |                | 
+| Dmitriy Ryaboy         | dvryaboy        | @dvryaboy           |                |
+| Jake Farrell           | jfarrell        |                     |                |
+| Jonathan Coveney       | jcoveney        | @jcoveney           |                |
+| Julien Le Dem          | julien          | @julienledem        | julienledem    |
+| Lukas Nalezenec        | lukas           | @lukasnalezenec     |                |
+| Marcel Kornacker       | marcel          | @mkornacker         |                |
+| Mickael Lacour         | mlacour         | @mickaellcr         |                |
+| Nong Li                | nong            | @nongli             |                |
+| Remy Pecqueur          | rpecqueur       | @Lordshinjo         |                |
+| Roman Shaposhnik       | rvs             | @rvs                |                |
+| Ryan Blue              | blue            | @rdblue             |                |
+| Sergio Pena            | spena           | @spena              | spena          |
+| Tianshuo Deng          | tianshuo        | @tsdeng             |                |
+| Todd Lipcon            | todd            | @toddlipcon         |                |
+| Tom White              | tomwhite        | @tomwhite           |                |
+| Wes McKinney           | wesm            | @wesm               |                |
+| Wesley Graham Peck     | wesleypeck      | @wesleypeck         |                |
+
+
+# Reviewing guidelines:
 Committers have the responsibility to give constructive and timely feedback on the pull requests.
 Anybody can give feedback on a pull request but only committers can merge it.
 

dev/merge_parquet_pr.py

@@ -81,9 +81,9 @@ def fail(msg):
 def run_cmd(cmd):
     try:       
         if isinstance(cmd, list):
-            return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            return subprocess.check_output(cmd)
         else:
-            return subprocess.check_output(cmd.split(" "), stderr = subprocess.STDOUT)          
+            return subprocess.check_output(cmd.split(" "))
     except subprocess.CalledProcessError as e:
         # this avoids hiding the stdout / stderr of failed processes
         print 'Command failed: %s' % cmd

parquet-avro/pom.xml

@@ -32,10 +32,6 @@
   <name>Apache Parquet Avro</name>
   <url>https://parquet.apache.org</url>
 
-  <properties>
-    <avro.version>1.7.6</avro.version>
-  </properties>
-
   <dependencies>
     <dependency>
       <groupId>org.apache.parquet</groupId>
@@ -71,7 +67,7 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>11.0</version>
+      <version>${guava.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -87,6 +83,13 @@
       <version>${slf4j.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-hadoop</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

parquet-avro/src/main/java/org/apache/parquet/avro/AvroIndexedRecordConverter.java

@@ -21,6 +21,8 @@
 import java.lang.reflect.Constructor;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.avro.Conversion;
+import org.apache.avro.LogicalType;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericArray;
 import org.apache.avro.generic.GenericData;
@@ -111,6 +113,11 @@ public void add(Object value) {
 
   @SuppressWarnings("unchecked")
   private static <T> Class<T> getDatumClass(GenericData model, Schema schema) {
+    if (model.getConversionFor(schema.getLogicalType()) != null) {
+      // use generic classes to pass data to conversions
+      return null;
+    }
+
     if (model instanceof SpecificData) {
       return (Class<T>) ((SpecificData) model).getClass(schema);
     }
@@ -133,7 +140,16 @@ private Schema.Field getAvroField(String parquetFieldName) {
   }
 
   private static Converter newConverter(Schema schema, Type type,
-      GenericData model, ParentValueContainer parent) {
+      GenericData model, ParentValueContainer setter) {
+
+    LogicalType logicalType = schema.getLogicalType();
+    // the expected type is always null because it is determined by the parent
+    // datum class, which never helps for generic. when logical types are added
+    // to specific, this should pass the expected type here.
+    Conversion<?> conversion = model.getConversionFor(logicalType);
+    ParentValueContainer parent = ParentValueContainer
+        .getConversionContainer(setter, conversion, schema);
+
     if (schema.getType().equals(Schema.Type.BOOLEAN)) {
       return new AvroConverters.FieldBooleanConverter(parent);
     } else if (schema.getType().equals(Schema.Type.INT)) {

parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java

@@ -110,9 +110,9 @@ public RecordMaterializer<T> prepareForRead(
     MessageType parquetSchema = readContext.getRequestedSchema();
     Schema avroSchema;
 
-    if (readContext.getReadSupportMetadata().get(AVRO_READ_SCHEMA_METADATA_KEY) != null) {
+    if (metadata.get(AVRO_READ_SCHEMA_METADATA_KEY) != null) {
       // use the Avro read schema provided by the user
-      avroSchema = new Schema.Parser().parse(readContext.getReadSupportMetadata().get(AVRO_READ_SCHEMA_METADATA_KEY));
+      avroSchema = new Schema.Parser().parse(metadata.get(AVRO_READ_SCHEMA_METADATA_KEY));
     } else if (keyValueMetaData.get(AVRO_SCHEMA_METADATA_KEY) != null) {
       // use the Avro schema from the file metadata if present
       avroSchema = new Schema.Parser().parse(keyValueMetaData.get(AVRO_SCHEMA_METADATA_KEY));