Skip to content

Commit 59025c8

Browse files
author
Davies Liu
committed
address comments from @marmbrus
1 parent 341ec2c commit 59025c8

File tree

2 files changed

+8
-3
lines changed

2 files changed

+8
-3
lines changed

sql/core/src/main/scala/org/apache/spark/sql/sources/DataSourceStrategy.scala

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._
2323
import org.apache.spark.sql.catalyst.planning.PhysicalOperation
2424
import org.apache.spark.sql.catalyst.plans.logical
2525
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
26+
import org.apache.spark.sql.execution.SparkPlan
2627
import org.apache.spark.sql.types.{UTF8String, StringType}
2728
import org.apache.spark.sql.{Row, Strategy, execution, sources}
2829

@@ -114,9 +115,10 @@ private[sql] object DataSourceStrategy extends Strategy {
114115
}
115116
}
116117

117-
private[this] def createPhysicalRDD(relation: BaseRelation,
118-
output: Seq[Attribute],
119-
rdd: RDD[Row]) = {
118+
private[this] def createPhysicalRDD(
119+
relation: BaseRelation,
120+
output: Seq[Attribute],
121+
rdd: RDD[Row]): SparkPlan = {
120122
val converted = if (relation.needConversion) {
121123
execution.RDDConversions.rowToRowRdd(rdd, relation.schema)
122124
} else {

sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,9 @@ abstract class BaseRelation {
131131
* Whether does it need to convert the objects in Row to internal representation, for example:
132132
* java.lang.String -> UTF8String
133133
* java.lang.Decimal -> Decimal
134+
*
135+
* Note: The internal representation is not stable across releases and thus data sources outside
136+
* of Spark SQL should leave this as true.
134137
*/
135138
def needConversion: Boolean = true
136139
}

0 commit comments

Comments
 (0)