Skip to content

Commit

Permalink
remove wrong compress type check (oap-project#1178)
Browse files Browse the repository at this point in the history
Since the compresssion has been supported in oap-project#1014 . The extra compression check in ArrowConvertorExtension can be remove now.
  • Loading branch information
jackylee-ch authored and zhouyuan committed Dec 14, 2022
1 parent 6ff98d9 commit ed16c96
Showing 1 changed file with 1 addition and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,7 @@ case class ArrowConvertorRule(session: SparkSession) extends Rule[LogicalPlan] {
case c: InsertIntoHadoopFsRelationCommand
if c.fileFormat.isInstanceOf[ParquetFileFormat] &&
c.partitionColumns.isEmpty && c.bucketSpec.isEmpty =>
// TODO: Support pass parquet config and writing with other codecs
// `compression`, `parquet.compression`(i.e., ParquetOutputFormat.COMPRESSION), and
// `spark.sql.parquet.compression.codec`
// are in order of precedence from highest to lowest.
val parquetCompressionConf = c.options.get(ParquetOutputFormat.COMPRESSION)
val codecName = c.options
.get("compression")
.orElse(parquetCompressionConf)
.getOrElse(session.sessionState.conf.parquetCompressionCodec)
.toLowerCase(Locale.ROOT)
if (codecName.equalsIgnoreCase("snappy")) {
c.copy(fileFormat = new ArrowFileFormat)
} else {
c
}
c.copy(fileFormat = new ArrowFileFormat)

// Read path
case l@ LogicalRelation(
Expand Down

0 comments on commit ed16c96

Please sign in to comment.