Skip to content

Commit ee70125

Browse files
fixing one problem with arrayconverter
1 parent 98219cf commit ee70125

File tree

2 files changed

+8
-5
lines changed

2 files changed

+8
-5
lines changed

sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetConverter.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,9 @@ class CatalystArrayConverter(
275275

276276
// TODO: think about reusing the buffer
277277
override def end(): Unit = {
278-
if (parent != null) parent.updateField(index, buffer)
278+
assert(parent != null)
279+
parent.updateField(index, buffer)
280+
clearBuffer()
279281
}
280282
}
281283

sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -405,11 +405,12 @@ class ParquetQuerySuite extends QueryTest with FunSuiteLike with BeforeAndAfterA
405405
assert(result(0)(3).asInstanceOf[ArrayBuffer[Any]].size === 2)
406406
assert(result(0)(3).asInstanceOf[ArrayBuffer[Any]].apply(0) === 2.5)
407407
assert(result(0)(3).asInstanceOf[ArrayBuffer[Any]].apply(1) === false)
408-
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].size === 3)
408+
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].size === 2)
409+
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].apply(0).asInstanceOf[ArrayBuffer[Any]].size === 2)
409410
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].apply(1).asInstanceOf[ArrayBuffer[Any]].size === 1)
410-
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].apply(1).asInstanceOf[ArrayBuffer[ArrayBuffer[Any]]].size === 1)
411-
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].apply(1).asInstanceOf[ArrayBuffer[ArrayBuffer[Any]]].apply(0) === 9)
412-
assert(true)
411+
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].apply(0).asInstanceOf[ArrayBuffer[ArrayBuffer[Any]]].apply(0).apply(0) === 7)
412+
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].apply(0).asInstanceOf[ArrayBuffer[ArrayBuffer[Any]]].apply(1).apply(0) === 8)
413+
assert(result(0)(4).asInstanceOf[ArrayBuffer[Any]].apply(1).asInstanceOf[ArrayBuffer[ArrayBuffer[Any]]].apply(0).apply(0) === 9)
413414
}
414415

415416
/**

0 commit comments

Comments
 (0)