Spark writing null complex types inside arrays and maps throw MatchError #2431
Description
When writing to ES with Spark, if you have null complex types inside of arrays or maps, a scala.MatchError: null, because nulls are never handled on this path:
scala.MatchError: null
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.writeStruct(DataFrameValueWriter.scala:89)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.write(DataFrameValueWriter.scala:116)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.$anonfun$doWriteSeq$1(DataFrameValueWriter.scala:139)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.$anonfun$doWriteSeq$1$adapted(DataFrameValueWriter.scala:135)
at app//scala.collection.IterableOnceOps.foreach(IterableOnce.scala:619)
at app//scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:617)
at app//scala.collection.AbstractIterable.foreach(Iterable.scala:935)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.doWriteSeq(DataFrameValueWriter.scala:135)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.writeArray(DataFrameValueWriter.scala:125)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.write(DataFrameValueWriter.scala:117)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.$anonfun$writeStruct$1(DataFrameValueWriter.scala:98)
at app//scala.collection.IterableOnceOps.foreach(IterableOnce.scala:619)
at app//scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:617)
at app//scala.collection.AbstractIterable.foreach(Iterable.scala:935)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.writeStruct(DataFrameValueWriter.scala:93)
at app//org.elasticsearch.spark.sql.DataFrameValueWriter.write(DataFrameValueWriter.scala:67)
at app//org.elasticsearch.spark.sql.DataFrameValueWriterTest.serialize(DataFrameValueWriterTest.scala:53)
at app//org.elasticsearch.spark.sql.DataFrameValueWriterTest.serialize(DataFrameValueWriterTest.scala:43)
at app//org.elasticsearch.spark.sql.DataFrameValueWriterTest.testNullStructInArray(DataFrameValueWriterTest.scala:163)