Skip to content

[SPARK-15698][SQL][Streaming][Follw-up]Fix FileStream source and sink log get configuration issue #15173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jerryshao wants to merge 1 commit into from
Closed

[SPARK-15698][SQL][Streaming][Follw-up]Fix FileStream source and sink log get configuration issue #15173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,11 @@ class FileStreamSinkLog(

private implicit val formats = Serialization.formats(NoTypeHints)

protected override val fileCleanupDelayMs =
sparkSession.conf.get(SQLConf.FILE_SINK_LOG_CLEANUP_DELAY)
protected override val fileCleanupDelayMs = sparkSession.sessionState.conf.fileSinkLogCleanupDelay

protected override val isDeletingExpiredLog =
sparkSession.conf.get(SQLConf.FILE_SINK_LOG_DELETION)
protected override val isDeletingExpiredLog = sparkSession.sessionState.conf.fileSinkLogDeletion

protected override val compactInterval =
sparkSession.conf.get(SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL)
protected override val compactInterval = sparkSession.sessionState.conf.fileSinkLogCompactInterval
require(compactInterval > 0,
s"Please set ${SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL.key} (was $compactInterval) " +
"to a positive value.")
Expand Down
7 changes: 3 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,15 @@ class FileStreamSourceLog(

// Configurations about metadata compaction
protected override val compactInterval =
sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_COMPACT_INTERVAL)
sparkSession.sessionState.conf.fileSourceLogCompactInterval
require(compactInterval > 0,
s"Please set ${SQLConf.FILE_SOURCE_LOG_COMPACT_INTERVAL.key} (was $compactInterval) to a " +
s"positive value.")

protected override val fileCleanupDelayMs =
sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_CLEANUP_DELAY)
sparkSession.sessionState.conf.fileSourceLogCleanupDelay

protected override val isDeletingExpiredLog =
sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_DELETION)
protected override val isDeletingExpiredLog = sparkSession.sessionState.conf.fileSourceLogDeletion

private implicit val formats = Serialization.formats(NoTypeHints)

Expand Down
8 changes: 7 additions & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -620,10 +620,16 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging {

def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION)

def fileSinkLogCompatInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL)
def fileSinkLogCompactInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL)

def fileSinkLogCleanupDelay: Long = getConf(FILE_SINK_LOG_CLEANUP_DELAY)

def fileSourceLogDeletion: Boolean = getConf(FILE_SOURCE_LOG_DELETION)

def fileSourceLogCompactInterval: Int = getConf(FILE_SOURCE_LOG_COMPACT_INTERVAL)

def fileSourceLogCleanupDelay: Long = getConf(FILE_SOURCE_LOG_CLEANUP_DELAY)

def streamingSchemaInference: Boolean = getConf(STREAMING_SCHEMA_INFERENCE)

def streamingPollingDelay: Long = getConf(STREAMING_POLLING_DELAY)
Expand Down