diff --git a/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala index 27a5b918fc94e..25e6aec4d84a5 100644 --- a/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala +++ b/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala @@ -182,16 +182,19 @@ private[sql] object AvroUtils extends Logging { def hasNextRow: Boolean = { while (!completed && currentRow.isEmpty) { - val r = fileReader.hasNext && !fileReader.pastSync(stopPosition) - if (!r) { + if (fileReader.pastSync(stopPosition)) { fileReader.close() completed = true currentRow = None - } else { + } else if (fileReader.hasNext()) { val record = fileReader.next() // the row must be deserialized in hasNextRow, because AvroDeserializer#deserialize // potentially filters rows currentRow = deserializer.deserialize(record).asInstanceOf[Option[InternalRow]] + } else { + // In this case, `fileReader.hasNext()` returns false but we are not past sync point yet. + // This means empty blocks, we need to continue reading the file in case there are non + // empty blocks or we are past sync point. } } currentRow.isDefined diff --git a/connector/avro/src/test/resources/empty_blocks.avro b/connector/avro/src/test/resources/empty_blocks.avro new file mode 100644 index 0000000000000..85d96f4af7158 --- /dev/null +++ b/connector/avro/src/test/resources/empty_blocks.avro @@ -0,0 +1,5 @@ +Obj6decoder.shape.version.patch0,decoder.shape.commitidP32d3df6520fbab9f829c638602bfcaf57a36af3a2decoder.shape.fingerprint18b0f3cb011ab5450a2eb866a48017b178f421c495cfea8e014a892a2e0af6c4$decoder.shape.usid ea88d6ea20b60173cmesg_shape.id118 cmesg_shape.name,testAvroMessageAAAAAAA(cmesg_shape.bytesize21.cmesg_shape.fingerprint5d0906d6748d14d2a4b65b5a18dc05ba39e2def91236c3d0d08c577afe38b0180decoder.software.version&cmessage-decoder==1_weoiwasd2weroqw_asdmjkjsdf_2p1_gccgggg.file.path/tmp/aaaaaa/bbbbbbbbbbb/ccccccccc/split-ggggs/5d0f6168_20231107134036/20231107161711.653/tmp_28dycwr-asd-ed-123-234-128-2.gggggggg.file.mode +batch gggg.file.st_ino(-4344181839388196375"gggg.file.st_size169201408$gggg.file.st_mtime.1998-02-15 16:26:43.000_col_15111.222.333.4 xyz_idHe4ed94eb-bfbd-458e-85af-cf1a7245a254 hrr_idhrr-5d0f6168 ofname20231107134036*gggg.decode_timestamp01998-02-15 16:28:02.264Zgggg.file.paths3://test-bucket-abcde/aa/bbbb/ccccccccccccc/ddddddddddddddddddd/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee/fffffffffffffffffffffff.gggg.hh"gggg.file.st_size40788641gggg.locations3://test-bucket-abcde/aa/bbbb/ccccccccccccc/ddddddddddddddddddd/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee/fffffffffffffffffffffff.gggg.hh2gggg.record_end_timestamp01998-02-15 13:45:35.000Z&gggg.xfer_timestamp01998-02-15 16:14:40.339Zgggg_bytes40788641port_idNoneraw>5d0f6168_20231107134036.gggg.gz(record_end_timestamp01998-02-15 13:45:35.000Zsplit_total2Pcmesg_shape.field.Column01xxxxxxxxxx.idx0Tcmesg_shape.field.Column01xxxxxxxxxx.ctypeInt32_tdcmesg_shape.field.Column01xxxxxxxxxx.struct_format>ijcmesg_shape.field.Column01xxxxxxxxxx.meameameameameamvalue0001\cmesg_shape.field.Column01xxxxxxxxxx.math_typevalue02tcmesg_shape.field.Column02xxxxxxxxxxxxxxxxxxxxxxxxxxxx.idx1xcmesg_shape.field.Column02xxxxxxxxxxxxxxxxxxxxxxxxxxxx.ctypeInt64_tcmesg_shape.field.Column02xxxxxxxxxxxxxxxxxxxxxxxxxxxx.struct_format>qcmesg_shape.field.Column02xxxxxxxxxxxxxxxxxxxxxxxxxxxx.meameameameameamvalue00000005cmesg_shape.field.Column02xxxxxxxxxxxxxxxxxxxxxxxxxxxx.math_typevalue02>cmesg_shape.field.Column03x.idx2Bcmesg_shape.field.Column03x.ctypeUint32_tRcmesg_shape.field.Column03x.struct_format>IXcmesg_shape.field.Column03x.meameameameameam eBytesJcmesg_shape.field.Column03x.math_typevalue02Bcmesg_shape.field.Column04xxx.idx3Fcmesg_shape.field.Column04xxx.ctypeboolVcmesg_shape.field.Column04xxx.struct_format>?\cmesg_shape.field.Column04xxx.meameameameameamvalue0001Ncmesg_shape.field.Column04xxx.math_typevalue004Dcmesg_shape.field.Column05xxxx.idx4Hcmesg_shape.field.Column05xxxx.ctypeUint32_tXcmesg_shape.field.Column05xxxx.struct_format>I^cmesg_shape.field.Column05xxxx.meameameameameam eBytesPcmesg_shape.field.Column05xxxx.math_typevalue02(gggg.first_timestamp01998-02-15 13:40:36.004ZZcmesg_shape.field.Column06xxxxxxxxxxxxxxx.idx5^cmesg_shape.field.Column06xxxxxxxxxxxxxxx.ctype2EControllableElementNamesncmesg_shape.field.Column06xxxxxxxxxxxxxxx.struct_format>Btcmesg_shape.field.Column06xxxxxxxxxxxxxxx.meameameameameamvalue0001fcmesg_shape.field.Column06xxxxxxxxxxxxxxx.math_typeNoneZcmesg_shape.field.Column07xxxxxxxxxxxxxxx.idx6^cmesg_shape.field.Column07xxxxxxxxxxxxxxx.ctype*Value0000000000000003ncmesg_shape.field.Column07xxxxxxxxxxxxxxx.struct_format>Btcmesg_shape.field.Column07xxxxxxxxxxxxxxx.meameameameameamvalue0001fcmesg_shape.field.Column07xxxxxxxxxxxxxxx.math_typeNoneRcmesg_shape.field.Column08xxxxxxxxxxx.idx7Vcmesg_shape.field.Column08xxxxxxxxxxx.ctype2EControllableElementNamesfcmesg_shape.field.Column08xxxxxxxxxxx.struct_format>Blcmesg_shape.field.Column08xxxxxxxxxxx.meameameameameamvalue0001^cmesg_shape.field.Column08xxxxxxxxxxx.math_typeNoneRcmesg_shape.field.Column09xxxxxxxxxxx.idx8Vcmesg_shape.field.Column09xxxxxxxxxxx.ctype*Value0000000000000003fcmesg_shape.field.Column09xxxxxxxxxxx.struct_format>Blcmesg_shape.field.Column09xxxxxxxxxxx.meameameameameamvalue0001^cmesg_shape.field.Column09xxxxxxxxxxx.math_typeNoneRcmesg_shape.field.Column10xxxxxxxxxxx.idx9Vcmesg_shape.field.Column10xxxxxxxxxxx.ctype2EControllableElementNamesfcmesg_shape.field.Column10xxxxxxxxxxx.struct_format>Blcmesg_shape.field.Column10xxxxxxxxxxx.meameameameameamvalue0001^cmesg_shape.field.Column10xxxxxxxxxxx.math_typeNoneRcmesg_shape.field.Column11xxxxxxxxxxx.idx10Vcmesg_shape.field.Column11xxxxxxxxxxx.ctype*Value0000000000000003fcmesg_shape.field.Column11xxxxxxxxxxx.struct_format>Blcmesg_shape.field.Column11xxxxxxxxxxx.meameameameameamvalue0001^cmesg_shape.field.Column11xxxxxxxxxxx.math_typeNoneavro.schema{"namespace": "aaaaaaaaaaaaaaaaaa.bbbb.cccc", "name": "testAvroMessageAAAAAAA", "type": "record", "fields": [{"name": "Column01xxxxxxxxxx", "type": "int"}, {"name": "Column02xxxxxxxxxxxxxxxxxxxxxxxxxxxx", "type": "long"}, {"name": "Column03x", "type": "long"}, {"name": "Column04xxx", "type": "boolean"}, {"name": "Column05xxxx", "type": "long"}, {"name": "Column06xxxxxxxxxxxxxxx", "type": {"type": "enum", "symbols": ["v_1_____", "v_2_____", "v_3_____________________", "v_4__________________", "v_5___", "v_6_____", "v_7__", "v_8____", "v_9______", "v_10________", "v_11___________", "v_12_______________", "v_13__________________", "v_14________", "v_15_", "v_16____________________", "v_17______", "v_18_____", "v_19______", "v_20__", "v_21______________", "v_22__", "v_23______________________", "v_24____________", "v_25___________", "v_26____________", "v_27_________________", "v_28_________________", "v_29______________________", "v_30______", "v_31_____B", "v_32_________", "v_33_________", "v_34___________qweroiqwer", "v_35________________________", "v_36________________________", "v_37____________________", "v_38_________", "v_39____________________", "v_40_____________________", "v_41______________________", "v_42_________", "v_43________", "v_44____________", "v_45_______________", "v_45_______________Basdflsdfqkl", "v_46___________", "v_47______________________", "v_48____________________", "v_49___________", "v_50___", "v_51_______", "v_52____________________", "v_31____________________", "v_59__", "v_60__________", "v_61___________", "v_62________________", "v_63____________________________", "v_64_____________________", "v_65____________", "v_66________________50___", "v_67_________50___", "v_68________________", "v_69____________________", "v_70___________", "v_71________", "v_72__________________", "v_73__________", "v_74_________", "v_129___________________UasdfsdAsdfl", "v_75_______________", "v_129_______________", "v_76______", "v_34___________UsdfIasdflkasDeweksdkjs", "v_77____________", "v_78____________________", "v_77__________________________", "v_78________________________________", "v_79________________________________", "v_80___________", "v_81_________________", "v_82_______________________", "v_83_______________________", "v_6_____qpwepaaaaa", "v_6_____lksdlkwerpiwoelk", "v_6_________________________23", "v_6_________________________45", "v_7__qpwepaaaaa", "v_7__lksdlkwerpiwoelk", "v_7______________________45", "v_7______________________56", "v_10________qpwepaaaaa", "v_10________lksdlkwerpiwoelk", "v_10_____________________________A", "v_10_____________________________B", "v_85_________", "v_85________________", "v_86_______________", "v_87__________________________", "v_88______________", "v_89_______________", "v_34___________wetuto", "v_34___________asdfasdf", "v_90______________________wetuto", "v_92___________", "v_91_____________________________", "v_93_______________", "v_90______________________asdfasdf", "v_94________________", "v_95__________asdfasdf", "v_32_________asfqwerlasdflkasdf", "v_71________PPOPOP", "v_96__________", "v_97______________", "v_98_____________________", "v_99_______", "v_100______________", "v_50___Bridge", "v_101__________", "v_102_______________________", "v_32_________exasdflk", "v_103___________________", "v_104___", "v_105_____________", "v_41______________________exasdflk", "v_106_________________", "v_107_____________", "v_108________________", "v_16____________________exasdflk", "v_109________", "v_110_______________________", "v_111____________", "v_112_________________", "v_113___________________", "v_105_____________UIasdfasdfasdf", "v_105_____________UIasdfwetuto", "v_114____________________", "v_115____", "v_116__________________", "v_117________________", "v_118_______", "v_6_____Taselkk", "v_7__Taselkk", "v_10________Taselkk", "v_119________________", "v_120_________________", "v_121__________________", "v_122________________________________", "v_123______", "v_123______qmasdfolkasdf", "v_124____________________", "v_125____________________", "v_126______________", "v_127___________________", "v_128_____________________", "v_129_____", "v_130_______________", "v_131___________", "v_62________________Pqweroi", "v_132__________________", "v_133___________________", "v_134_________________", "v_129_____bnasdkwel", "v_170_________", "v_171_________________________________", "v_6_____exasdflk", "v_7__exasdflk", "v_10________exasdflk", "v_135________________", "v_136_________________", "v_137_________________", "v_32_________qwerUasdIlasdlKsdflakslksdk", "v_138_________________________", "v_139_______________________", "v_136_________________exasdflk", "v_140_____________", "v_141_____", "v_142_____", "v_143_____", "v_144________________________", "v_145_________________", "v_146___________________", "v_147_______________________________", "v_148_________", "v_149__________________", "v_150________________________", "v_151_____________________", "v_25___________lkxcvmnsmljnnj", "v_152______________________", "v_153______________________", "v_154_____________________________________________", "v_136_________________wetuto", "v_155________", "v_156_________", "v_25___________exasdflk", "v_157________________________", "v_158___________________________________", "v_159___________________", "v_104___Oqwe", "v_160_________________________", "v_161__", "v_162______", "v_163_________________________", "v_164____________", "bcount"], "name": "_po_asdfadslka_asxqweoioasdflkasdfkjklas"}}, {"name": "Column07xxxxxxxxxxxxxxx", "type": {"type": "enum", "symbols": ["v_1_____", "eFso", "eFsi", "iOIJ", "iOIK", "eS0", "eS1", "eS2", "eP0", "eP1", "eP2", "eRp", "eRs", "eRs0", "eRs1", "eRs2", "eRp0", "eRp1", "eRp2", "v_124____0", "v_124____1", "v_124____2", "v_124____3", "v_124____4", "v_124____5", "v_125__________", "e1", "e1A", "e1B", "e2", "e2A", "e2B", "e3", "e3A", "e3B", "e4", "e4A", "e4B", "e5", "e5A", "e5B", "e6", "e6A", "e6B", "e7", "e8", "e9", "e10", "e11", "e12", "e13", "e14", "v_126_________0", "v_127_________", "v_128______________________________", "v_129___________________Bqweroiqwer", "v_129___________________Cqweroiqwer", "v_1300", "v_1301", "v_131_____0", "e0", "v_126_________1", "v_126_________2", "v_129_______________", "v_132________", "v_133_________", "v_130Raex", "eA", "eB", "bgrtasd", "bcount"], "name": "_aa_bbbbbbbbbb_cccccc0000000000000003"}}, {"name": "Column08xxxxxxxxxxx", "type": {"type": "enum", "symbols": ["v_1_____", "v_2_____", "v_3_____________________", "v_4__________________", "v_5___", "v_6_____", "v_7__", "v_8____", "v_9______", "v_10________", "v_11___________", "v_12_______________", "v_13__________________", "v_14________", "v_15_", "v_16____________________", "v_17______", "v_18_____", "v_19______", "v_20__", "v_21______________", "v_22__", "v_23______________________", "v_24____________", "v_25___________", "v_26____________", "v_27_________________", "v_28_________________", "v_29______________________", "v_30______", "v_31_____B", "v_32_________", "v_33_________", "v_34___________qweroiqwer", "v_35________________________", "v_36________________________", "v_37____________________", "v_38_________", "v_39____________________", "v_40_____________________", "v_41______________________", "v_42_________", "v_43________", "v_44____________", "v_45_______________", "v_45_______________Basdflsdfqkl", "v_46___________", "v_47______________________", "v_48____________________", "v_49___________", "v_50___", "v_51_______", "v_52____________________", "v_31____________________", "v_59__", "v_60__________", "v_61___________", "v_62________________", "v_63____________________________", "v_64_____________________", "v_65____________", "v_66________________50___", "v_67_________50___", "v_68________________", "v_69____________________", "v_70___________", "v_71________", "v_72__________________", "v_73__________", "v_74_________", "v_129___________________UasdfsdAsdfl", "v_75_______________", "v_129_______________", "v_76______", "v_34___________UsdfIasdflkasDeweksdkjs", "v_77____________", "v_78____________________", "v_77__________________________", "v_78________________________________", "v_79________________________________", "v_80___________", "v_81_________________", "v_82_______________________", "v_83_______________________", "v_6_____qpwepaaaaa", "v_6_____lksdlkwerpiwoelk", "v_6___________________________", "v_6__________________________1", "v_7__qpwepaaaaa", "v_7__lksdlkwerpiwoelk", "v_7______________________78", "v_7________________________", "v_10________qpwepaaaaa", "v_10________lksdlkwerpiwoelk", "v_10_____________________________C", "v_10_____________________________D", "v_85_________", "v_85________________", "v_86_______________", "v_87__________________________", "v_88______________", "v_89_______________", "v_34___________wetuto", "v_34___________asdfasdf", "v_90______________________wetuto", "v_92___________", "v_91_____________________________", "v_93_______________", "v_90______________________asdfasdf", "v_94________________", "v_95__________asdfasdf", "v_32_________asfqwerlasdflkasdf", "v_71________PPOPOP", "v_96__________", "v_97______________", "v_98_____________________", "v_99_______", "v_100______________", "v_50___Bridge", "v_101__________", "v_102_______________________", "v_32_________exasdflk", "v_103___________________", "v_104___", "v_105_____________", "v_41______________________exasdflk", "v_106_________________", "v_107_____________", "v_108________________", "v_16____________________exasdflk", "v_109________", "v_110_______________________", "v_111____________", "v_112_________________", "v_113___________________", "v_105_____________UIasdfasdfasdf", "v_105_____________UIasdfwetuto", "v_114____________________", "v_115____", "v_116__________________", "v_117________________", "v_118_______", "v_6_____Taselkk", "v_7__Taselkk", "v_10________Taselkk", "v_119________________", "v_120_________________", "v_121__________________", "v_122________________________________", "v_123______", "v_123______qmasdfolkasdf", "v_124____________________", "v_125____________________", "v_126______________", "v_127___________________", "v_128_____________________", "v_129_____", "v_130_______________", "v_131___________", "v_62________________Pqweroi", "v_132__________________", "v_133___________________", "v_134_________________", "v_129_____bnasdkwel", "v_170_________", "v_171_________________________________", "v_6_____exasdflk", "v_7__exasdflk", "v_10________exasdflk", "v_135________________", "v_136_________________", "v_137_________________", "v_32_________qwerUasdIlasdlKsdflakslksdk", "v_138_________________________", "v_139_______________________", "v_136_________________exasdflk", "v_140_____________", "v_141_____", "v_142_____", "v_143_____", "v_144________________________", "v_145_________________", "v_146___________________", "v_147_______________________________", "v_148_________", "v_149__________________", "v_150________________________", "v_151_____________________", "v_25___________lkxcvmnsmljnnj", "v_152______________________", "v_153______________________", "v_154_____________________________________________", "v_136_________________wetuto", "v_155________", "v_156_________", "v_25___________exasdflk", "v_157________________________", "v_158___________________________________", "v_159___________________", "v_104___Oqwe", "v_160_________________________", "v_161__", "v_162______", "v_163_________________________", "v_164____________", "bcount"], "name": "_jk_wew_sd_asdflkweroiuwerlkasdfljas"}}, {"name": "Column09xxxxxxxxxxx", "type": {"type": "enum", "symbols": ["v_1_____", "eFso", "eFsi", "iOIJ", "iOIK", "eS0", "eS1", "eS2", "eP0", "eP1", "eP2", "eRp", "eRs", "eRs0", "eRs1", "eRs2", "eRp0", "eRp1", "eRp2", "v_124____0", "v_124____1", "v_124____2", "v_124____3", "v_124____4", "v_124____5", "v_125__________", "e1", "e1A", "e1B", "e2", "e2A", "e2B", "e3", "e3A", "e3B", "e4", "e4A", "e4B", "e5", "e5A", "e5B", "e6", "e6A", "e6B", "e7", "e8", "e9", "e10", "e11", "e12", "e13", "e14", "v_126_________0", "v_127_________", "v_128______________________________", "v_129___________________Bqweroiqwer", "v_129___________________Cqweroiqwer", "v_1300", "v_1301", "v_131_____0", "e0", "v_126_________1", "v_126_________2", "v_129_______________", "v_132________", "v_133_________", "v_130Raex", "eA", "eB", "bgrtasd", "bcount"], "name": "_tt_ppp_xx_yyyyyy0000000000000003"}}, {"name": "Column10xxxxxxxxxxx", "type": {"type": "enum", "symbols": ["v_1_____", "v_2_____", "v_3_____________________", "v_4__________________", "v_5___", "v_6_____", "v_7__", "v_8____", "v_9______", "v_10________", "v_11___________", "v_12_______________", "v_13__________________", "v_14________", "v_15_", "v_16____________________", "v_17______", "v_18_____", "v_19______", "v_20__", "v_21______________", "v_22__", "v_23______________________", "v_24____________", "v_25___________", "v_26____________", "v_27_________________", "v_28_________________", "v_29______________________", "v_30______", "v_31_____B", "v_32_________", "v_33_________", "v_34___________qweroiqwer", "v_35________________________", "v_36________________________", "v_37____________________", "v_38_________", "v_39____________________", "v_40_____________________", "v_41______________________", "v_42_________", "v_43________", "v_44____________", "v_45_______________", "v_45_______________Basdflsdfqkl", "v_46___________", "v_47______________________", "v_48____________________", "v_49___________", "v_50___", "v_51_______", "v_52____________________", "v_31____________________", "v_59__", "v_60__________", "v_61___________", "v_62________________", "v_63____________________________", "v_64_____________________", "v_65____________", "v_66________________50___", "v_67_________50___", "v_68________________", "v_69____________________", "v_70___________", "v_71________", "v_72__________________", "v_73__________", "v_74_________", "v_129___________________UasdfsdAsdfl", "v_75_______________", "v_129_______________", "v_76______", "v_34___________UsdfIasdflkasDeweksdkjs", "v_77____________", "v_78____________________", "v_77__________________________", "v_78________________________________", "v_79________________________________", "v_80___________", "v_81_________________", "v_82_______________________", "v_83_______________________", "v_6_____qpwepaaaaa", "v_6_____lksdlkwerpiwoelk", "v_6_________________________34", "v_6_________________________12", "v_7__qpwepaaaaa", "v_7__lksdlkwerpiwoelk", "v_7______________________12", "v_7______________________34", "v_10________qpwepaaaaa", "v_10________lksdlkwerpiwoelk", "v_10_____________________________E", "v_10_____________________________F", "v_85_________", "v_85________________", "v_86_______________", "v_87__________________________", "v_88______________", "v_89_______________", "v_34___________wetuto", "v_34___________asdfasdf", "v_90______________________wetuto", "v_92___________", "v_91_____________________________", "v_93_______________", "v_90______________________asdfasdf", "v_94________________", "v_95__________asdfasdf", "v_32_________asfqwerlasdflkasdf", "v_71________PPOPOP", "v_96__________", "v_97______________", "v_98_____________________", "v_99_______", "v_100______________", "v_50___Bridge", "v_101__________", "v_102_______________________", "v_32_________exasdflk", "v_103___________________", "v_104___", "v_105_____________", "v_41______________________exasdflk", "v_106_________________", "v_107_____________", "v_108________________", "v_16____________________exasdflk", "v_109________", "v_110_______________________", "v_111____________", "v_112_________________", "v_113___________________", "v_105_____________UIasdfasdfasdf", "v_105_____________UIasdfwetuto", "v_114____________________", "v_115____", "v_116__________________", "v_117________________", "v_118_______", "v_6_____Taselkk", "v_7__Taselkk", "v_10________Taselkk", "v_119________________", "v_120_________________", "v_121__________________", "v_122________________________________", "v_123______", "v_123______qmasdfolkasdf", "v_124____________________", "v_125____________________", "v_126______________", "v_127___________________", "v_128_____________________", "v_129_____", "v_130_______________", "v_131___________", "v_62________________Pqweroi", "v_132__________________", "v_133___________________", "v_134_________________", "v_129_____bnasdkwel", "v_170_________", "v_171_________________________________", "v_6_____exasdflk", "v_7__exasdflk", "v_10________exasdflk", "v_135________________", "v_136_________________", "v_137_________________", "v_32_________qwerUasdIlasdlKsdflakslksdk", "v_138_________________________", "v_139_______________________", "v_136_________________exasdflk", "v_140_____________", "v_141_____", "v_142_____", "v_143_____", "v_144________________________", "v_145_________________", "v_146___________________", "v_147_______________________________", "v_148_________", "v_149__________________", "v_150________________________", "v_151_____________________", "v_25___________lkxcvmnsmljnnj", "v_152______________________", "v_153______________________", "v_154_____________________________________________", "v_136_________________wetuto", "v_155________", "v_156_________", "v_25___________exasdflk", "v_157________________________", "v_158___________________________________", "v_159___________________", "v_104___Oqwe", "v_160_________________________", "v_161__", "v_162______", "v_163_________________________", "v_164____________", "bcount"], "name": "_we_asd_qw_qwerplksdflksfwerijllsadk"}}, {"name": "Column11xxxxxxxxxxx", "type": {"type": "enum", "symbols": ["v_1_____", "eFso", "eFsi", "iOIJ", "iOIK", "eS0", "eS1", "eS2", "eP0", "eP1", "eP2", "eRp", "eRs", "eRs0", "eRs1", "eRs2", "eRp0", "eRp1", "eRp2", "v_124____0", "v_124____1", "v_124____2", "v_124____3", "v_124____4", "v_124____5", "v_125__________", "e1", "e1A", "e1B", "e2", "e2A", "e2B", "e3", "e3A", "e3B", "e4", "e4A", "e4B", "e5", "e5A", "e5B", "e6", "e6A", "e6B", "e7", "e8", "e9", "e10", "e11", "e12", "e13", "e14", "v_126_________0", "v_127_________", "v_128______________________________", "v_129___________________Bqweroiqwer", "v_129___________________Cqweroiqwer", "v_1300", "v_1301", "v_131_____0", "e0", "v_126_________1", "v_126_________2", "v_129_______________", "v_132________", "v_133_________", "v_130Raex", "eA", "eB", "bgrtasd", "bcount"], "name": "_aa_bbb_cc_dddddd0000000000000003"}}, {"name": "Column12xxxxxxx", "type": {"type": "long", "logicalType": "timestamp-micros"}}, {"name": "Column13xx", "type": "string"}, {"name": "Column14xx", "type": "string"}, {"name": "_col_15", "type": "string"}, {"name": "_col_16", "type": "string"}, {"name": "_col_17___", "type": "int"}, {"name": "_col_18___", "type": "int"}, {"name": "_col_19___", "type": "int"}, {"name": "_col_20_____", "type": "long"}, {"name": "_col_20_____2", "type": "int"}, {"name": "_col_22________________", "type": "long"}]}avro.codecdeflate@fi+)wd5- +@fi+)wd5-cX3uL &&&j[;YK/mw|eu.3Оu쥊=Pl98gZ7 GÄO"yg ׼p=&j6Pw,gYض@fi+)wd5-caL &&&j^?M))*)0P<***9EHBTPgf$CUC;Y>"Ypg:U122,139%wνB=|p# - Qoncʹ>#ǃTgo1cr${L=H {6}mpݙ5n/N=k`lyf4$kVXA k`˂uԾ +aϮ3O?Rўe',yQh̲(?2@fi+)wd5-cUL &&&jfM))*)0P<***9EHBTPgf$CE=, a`hZAk&=f&3ȹ{\D~<@fi+)wd5-cuuUL &&&j۶im}¦hdejeihe``e`dd`elehbhdfi"ehighfghbg(33h!~sϢ]אa`-5122tTM10_/7S;p=k6Sw.dy3Sy@fi+)wd5-cXȺUL && _~h}¦hdejeihe``e`dd`elehbhdfi"ehighfghbg(322Y10xdڵT̠Ir?p0߽7[DEܛ~>PC=&j]Sw&=d9rE y@fi+)wd5-cUL && 35i}¦hdejeihe``e`dd`elehbhdfi"ehighfghbg(322Lg `XlZC_ 5143h!m 㺧=, O=䝆'R;=V{g1${LԾ=Pa 읶GG"Y3Գ],| a`*BJ?S, tGgʼ='h g3 0tCD&X okf#Ĺ@fi+)wd5-c3܋UL && 5\n}¦hdejeihe``e`dd`elehbhdfi"ehighfghbg(32230CDHO#5143h!~dU, w-dM+wgͺӌsv;RXƼ}fy@fi+)wd5-cXpUL &&&j s"$maxPartitionBytes") { + val file = getResourceAvroFilePath("empty_blocks.avro") + val df = spark.read.format("avro").load(file) + val count = df.count() + val records = df.collect() + assert(count == 58) + assert(count == records.length) + } + } + } } class AvroV1Suite extends AvroSuite {