diff --git a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala new file mode 100644 index 0000000000..306b1df4f6 --- /dev/null +++ b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.sql.datasources.shapefile + +import org.apache.spark.Partition +import org.apache.spark.sql.connector.read.InputPartition +import org.apache.spark.sql.execution.datasources.PartitionedFile + +case class ShapefilePartition(index: Int, files: Array[PartitionedFile]) + extends Partition + with InputPartition diff --git a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala index ba25c92dad..5a28af6d66 100644 --- a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala +++ b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala @@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader import org.apache.spark.sql.connector.read.PartitionReaderFactory import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues -import org.apache.spark.sql.execution.datasources.FilePartition import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType @@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory( override def createReader(partition: InputPartition): PartitionReader[InternalRow] = { partition match { - case filePartition: FilePartition => buildReader(filePartition.files) + case filePartition: ShapefilePartition => buildReader(filePartition.files) case _ => throw new IllegalArgumentException( s"Unexpected partition type: ${partition.getClass.getCanonicalName}") diff --git a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala index f8f4cac2f0..afdface894 100644 --- a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala +++ b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala @@ -105,7 +105,7 @@ case class ShapefileScan( } else false } if (!hasMissingFiles) { - Some(FilePartition(index, group.values.toArray)) + Some(ShapefilePartition(index, group.values.toArray)) } else { None }