Fix files truncating according to maxRecordPerFile (databricks#180)

* Fix files truncating according to maxRecordPerFile * toDouble
datamechanics · May 29, 2019 · 6b2bf9f · 6b2bf9f
1 parent 3f92a09
commit 6b2bf9f
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/main/scala/com/databricks/spark/sql/perf/Tables.scala b/src/main/scala/com/databricks/spark/sql/perf/Tables.scala
@@ -222,7 +222,7 @@ abstract class Tables(sqlContext: SQLContext, scaleFactor: String,
           log.info(s"Data has $numRows rows clustered $clusterByPartitionColumns for $maxRecordPerFile")
 
           if (maxRecordPerFile > 0 && numRows > maxRecordPerFile) {
-            val numFiles = ((numRows)/maxRecordPerFile).ceil.toInt
+            val numFiles = (numRows.toDouble/maxRecordPerFile).ceil.toInt
             println(s"Coalescing into $numFiles files")
             log.info(s"Coalescing into $numFiles files")
             data.coalesce(numFiles).write