From 35ea67065a02aa1cbf7fe3d4c0dba6b80590974a Mon Sep 17 00:00:00 2001 From: Tom White Date: Thu, 30 Oct 2014 13:40:31 +0000 Subject: [PATCH 1/2] PARQUET-121. Allow Parquet to build with Java 8. --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1293a5d5c9..e36b77cbc7 100644 --- a/pom.xml +++ b/pom.xml @@ -75,8 +75,9 @@ + 1.6 1.6 - 1.6 + ${targetJavaVersion} UTF-8 github 2.4 @@ -318,7 +319,7 @@ -XX:MaxPermSize=256m 1.6 - 1.6 + ${targetJavaVersion} UTF-8 From 8717e13ad48a92645c3eaf098529c66805ce0de7 Mon Sep 17 00:00:00 2001 From: Tom White Date: Tue, 28 Oct 2014 16:21:21 +0000 Subject: [PATCH 2/2] Fix tests to run under Java 8. --- .../parquet/pig/convert/MapConverter.java | 3 +- .../java/parquet/pig/TestParquetStorer.java | 20 ++++++++-- .../parquet/pig/TestPigSchemaConverter.java | 7 +++- parquet-scrooge/pom.xml | 5 ++- .../thrift/TestParquetWriteProtocol.java | 39 ++++++++++++++++++- pom.xml | 1 + 6 files changed, 66 insertions(+), 9 deletions(-) diff --git a/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java b/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java index 2827239c5a..81220e1894 100644 --- a/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java +++ b/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -78,7 +79,7 @@ final public void start() { @Override public void end() { - parent.add(new HashMap(buffer)); + parent.add(new LinkedHashMap(buffer)); } /** diff --git a/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java b/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java index 99cd6a2372..9d3222685b 100644 --- a/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java +++ b/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java @@ -121,16 +121,28 @@ public void testMultipleSchema() throws ExecException, Exception { final Schema schema = data.getSchema("out"); assertEquals(2, schema.size()); - assertEquals("a", schema.getField(0).alias); - assertEquals("b", schema.getField(1).alias); + // union could be in either order + int ai; + int bi; + if ("a".equals(schema.getField(0).alias)) { + ai = 0; + bi = 1; + assertEquals("a", schema.getField(0).alias); + assertEquals("b", schema.getField(1).alias); + } else { + ai = 1; + bi = 0; + assertEquals("b", schema.getField(0).alias); + assertEquals("a", schema.getField(1).alias); + } assertEquals(rows * 2, result.size()); int a = 0; int b = 0; for (Tuple tuple : result) { - String fa = (String) tuple.get(0); - String fb = (String) tuple.get(1); + String fa = (String) tuple.get(ai); + String fb = (String) tuple.get(bi); if (fa != null) { assertEquals("a" + a, fa); ++a; diff --git a/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java b/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java index e95150fbca..86156025e4 100644 --- a/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java +++ b/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java @@ -22,11 +22,14 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.util.Utils; +import org.junit.Ignore; import org.junit.Test; import parquet.schema.MessageType; @@ -214,8 +217,8 @@ public void testAnnonymousField() throws Exception { @Test public void testSchemaEvolution() { - Map> map = new HashMap>(); - map.put("pig.schema", new HashSet(Arrays.asList( + Map> map = new LinkedHashMap>(); + map.put("pig.schema", new LinkedHashSet(Arrays.asList( "a:int, b:int, c:int, d:int, e:int, f:int", "aa:int, aaa:int, b:int, c:int, ee:int"))); Schema result = getPigSchemaFromMultipleFiles(new MessageType("empty"), map); diff --git a/parquet-scrooge/pom.xml b/parquet-scrooge/pom.xml index 341a804be1..035a0acc32 100644 --- a/parquet-scrooge/pom.xml +++ b/parquet-scrooge/pom.xml @@ -128,7 +128,7 @@ net.alchim31.maven scala-maven-plugin - 3.0.1 + 3.1.6 scala-compile-first @@ -146,6 +146,9 @@ + + ${scala.maven.test.skip} + diff --git a/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java b/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java index 4c211cd217..bea5bafc82 100644 --- a/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java +++ b/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.TreeMap; +import org.junit.ComparisonFailure; import thrift.test.OneOfEach; import org.apache.pig.data.Tuple; @@ -95,12 +96,48 @@ public void testMap() throws Exception { "endField(names, 1)", "endMessage()" }; + String[] expectationsAlt = { + "startMessage()", + "startField(name, 0)", + "addBinary(map_name)", + "endField(name, 0)", + "startField(names, 1)", + "startGroup()", + "startField(map, 0)", + "startGroup()", + "startField(key, 0)", + "addBinary(foo2)", + "endField(key, 0)", + "startField(value, 1)", + "addBinary(bar2)", + "endField(value, 1)", + "endGroup()", + "startGroup()", + "startField(key, 0)", + "addBinary(foo)", + "endField(key, 0)", + "startField(value, 1)", + "addBinary(bar)", + "endField(value, 1)", + "endGroup()", + "endField(map, 0)", + "endGroup()", + "endField(names, 1)", + "endMessage()" + }; final Map map = new TreeMap(); map.put("foo", "bar"); map.put("foo2", "bar2"); TestMap testMap = new TestMap("map_name", map); - validatePig(expectations, testMap); + try { + validatePig(expectations, testMap); + } catch (ComparisonFailure e) { + // This can happen despite using a stable TreeMap, since ThriftToPig#toPigMap + // in com.twitter.elephantbird.pig.util creates a HashMap. + // So we test with the map elements in reverse order + validatePig(expectationsAlt, testMap); + } validateThrift(expectations, testMap); } diff --git a/pom.xml b/pom.xml index e36b77cbc7..95bacea612 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ 2.10.4 2.10 + false 0.11.1