Skip to content

Commit

Permalink
Backport to Spark 3.0 to 3.4
Browse files Browse the repository at this point in the history
  • Loading branch information
@Kontinuation
Kontinuation committed Sep 2, 2024
1 parent ad15f9d commit 498abdc
Show file tree
Hide file tree
Showing 10 changed files with 10 additions and 15 deletions.
3 changes: 1 addition & 2 deletions ...n/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
Expand Down Expand Up @@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(

override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
partition match {
case filePartition: FilePartition => buildReader(filePartition.files)
case filePartition: ShapefilePartition => buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
Expand Down
2 changes: 1 addition & 1 deletion .../spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
Some(FilePartition(index, group.values.toArray))
Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
Expand Down
3 changes: 1 addition & 2 deletions ...n/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
Expand Down Expand Up @@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(

override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
partition match {
case filePartition: FilePartition => buildReader(filePartition.files)
case filePartition: ShapefilePartition => buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
Expand Down
2 changes: 1 addition & 1 deletion .../spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
Some(FilePartition(index, group.values.toArray))
Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
Expand Down
3 changes: 1 addition & 2 deletions ...n/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
Expand Down Expand Up @@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(

override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
partition match {
case filePartition: FilePartition => buildReader(filePartition.files)
case filePartition: ShapefilePartition => buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
Expand Down
2 changes: 1 addition & 1 deletion .../spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
Some(FilePartition(index, group.values.toArray))
Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
Expand Down
3 changes: 1 addition & 2 deletions ...n/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
Expand Down Expand Up @@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(

override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
partition match {
case filePartition: FilePartition => buildReader(filePartition.files)
case filePartition: ShapefilePartition => buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
Expand Down
2 changes: 1 addition & 1 deletion .../spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
Some(FilePartition(index, group.values.toArray))
Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
Expand Down
3 changes: 1 addition & 2 deletions ...n/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
Expand Down Expand Up @@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(

override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
partition match {
case filePartition: FilePartition => buildReader(filePartition.files)
case filePartition: ShapefilePartition => buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
Expand Down
2 changes: 1 addition & 1 deletion .../spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
Some(FilePartition(index, group.values.toArray))
Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
Expand Down

0 comments on commit 498abdc

Please sign in to comment.