From 184c59404a229838ae10e1a102a2be23386cbc1c Mon Sep 17 00:00:00 2001 From: wangfei Date: Tue, 2 Dec 2014 12:42:08 +0800 Subject: [PATCH 1/4] using executeCollect instead collect --- .../src/main/scala/org/apache/spark/sql/hive/HiveContext.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index 304b9a73ee91d..b7dd406dffc85 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -377,7 +377,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) { command.executeCollect().map(_.head.toString) case other => - val result: Seq[Seq[Any]] = toRdd.map(_.copy()).collect().toSeq + val result: Seq[Seq[Any]] = other.executeCollect().toSeq // We need the types so we can output struct field names val types = analyzed.output.map(_.dataType) // Reformat to match hive tab delimited output. From 0db7ce8f132a4ebf1d83bbf1ab97f81bc812b854 Mon Sep 17 00:00:00 2001 From: wangfei Date: Tue, 2 Dec 2014 20:34:45 +0800 Subject: [PATCH 2/4] adding debug info --- .../src/main/scala/org/apache/spark/sql/hive/HiveContext.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index b7dd406dffc85..5ba5d315d831f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -417,6 +417,9 @@ object HiveContext { case (decimal: Decimal, DecimalType()) => // Hive strips trailing zeros so use its toString HiveShim.createDecimal(decimal.toBigDecimal.underlying()).toString case (other, tpe) if primitiveTypes contains tpe => other.toString + case (a, b) => println(a.getClass); println(b.getClass); + println(a.toString + " " + b.toString); + a.toString + " " + b.toString; } /** Hive outputs fields of structs slightly differently than top level attributes. */ From a60d6808438f7fac0e94c5740086ff83e8387446 Mon Sep 17 00:00:00 2001 From: wangfei Date: Tue, 2 Dec 2014 21:13:07 +0800 Subject: [PATCH 3/4] fix test failure --- .../src/main/scala/org/apache/spark/sql/hive/HiveContext.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index 5ba5d315d831f..39506f7a4619f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -416,6 +416,8 @@ object HiveContext { case (bin: Array[Byte], BinaryType) => new String(bin, "UTF-8") case (decimal: Decimal, DecimalType()) => // Hive strips trailing zeros so use its toString HiveShim.createDecimal(decimal.toBigDecimal.underlying()).toString + case (decimal: BigDecimal, DecimalType()) => + HiveShim.createDecimal(decimal.underlying()).toString case (other, tpe) if primitiveTypes contains tpe => other.toString case (a, b) => println(a.getClass); println(b.getClass); println(a.toString + " " + b.toString); From a5ab68ecf599328630aa18e6403a02e8b56f1dbe Mon Sep 17 00:00:00 2001 From: wangfei Date: Tue, 2 Dec 2014 21:19:06 +0800 Subject: [PATCH 4/4] Revert "adding debug info" This reverts commit 0db7ce8f132a4ebf1d83bbf1ab97f81bc812b854. --- .../src/main/scala/org/apache/spark/sql/hive/HiveContext.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index 39506f7a4619f..34fc21e61f60f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -419,9 +419,6 @@ object HiveContext { case (decimal: BigDecimal, DecimalType()) => HiveShim.createDecimal(decimal.underlying()).toString case (other, tpe) if primitiveTypes contains tpe => other.toString - case (a, b) => println(a.getClass); println(b.getClass); - println(a.toString + " " + b.toString); - a.toString + " " + b.toString; } /** Hive outputs fields of structs slightly differently than top level attributes. */