Skip to content

Commit

Permalink
[Jenkins,Spark] Stop running Nexmark suite for deprecated Spark 2 run…
Browse files Browse the repository at this point in the history 
…ner and ensure consistent version of Spark is used for Nexmark tests (fixes #23634)
  • Loading branch information
Moritz Mack committed Oct 14, 2022
1 parent 7e7cbd6 commit c2aa228
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 93 deletions.
8 changes: 6 additions & 2 deletions .test-infra/jenkins/CommonTestProperties.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ class CommonTestProperties {
return "1.13"
}

static String getSparkVersion() {
return "3"
}

enum Runner {
DATAFLOW("DataflowRunner"),
TEST_DATAFLOW("TestDataflowRunner"),
Expand All @@ -42,8 +46,8 @@ class CommonTestProperties {
JAVA: [
DATAFLOW: ":runners:google-cloud-dataflow-java",
TEST_DATAFLOW: ":runners:google-cloud-dataflow-java",
SPARK: ":runners:spark:2",
SPARK_STRUCTURED_STREAMING: ":runners:spark:2",
SPARK: ":runners:spark:${CommonTestProperties.getSparkVersion()}",
SPARK_STRUCTURED_STREAMING: ":runners:spark:${CommonTestProperties.getSparkVersion()}",
FLINK: ":runners:flink:${CommonTestProperties.getFlinkVersion()}",
DIRECT: ":runners:direct-java"
],
Expand Down
106 changes: 15 additions & 91 deletions .test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_

// Gradle goals for this job.
steps {
shell('echo "*** RUN NEXMARK IN BATCH MODE USING SPARK 2 RUNNER ***"')
shell("echo \"*** RUN NEXMARK IN BATCH MODE USING SPARK ${CommonTestProperties.getSparkVersion()} RUNNER ***\"")
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":runners:spark:2"' +
switches("-Pnexmark.runner=\":runners:spark:${CommonTestProperties.getSparkVersion()}\"" +
' -Pnexmark.args="' +
[
commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
Expand All @@ -55,14 +55,14 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
'--streamTimeout=60' ,
'--manageResources=false',
'--monitorJobs=true'
].join(' '))
].join(' ') + '"')
}
shell('echo "*** RUN NEXMARK SQL IN BATCH MODE USING SPARK 2 RUNNER ***"')
shell("echo \"*** RUN NEXMARK SQL IN BATCH MODE USING SPARK ${CommonTestProperties.getSparkVersion()} RUNNER ***\"")
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":runners:spark:2"' +
switches("-Pnexmark.runner=\":runners:spark:${CommonTestProperties.getSparkVersion()}\"" +
' -Pnexmark.args="' +
[
commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
Expand All @@ -71,54 +71,17 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
'--queryLanguage=sql',
'--streaming=false',
'--suite=SMOKE',
'--streamTimeout=60' ,
'--manageResources=false',
'--monitorJobs=true'
].join(' '))
}
shell('echo "*** RUN NEXMARK IN BATCH MODE USING SPARK 3 RUNNER ***"')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":runners:spark:3"' +
' -Pnexmark.args="' +
[
commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
'--runner=SparkRunner',
'--streaming=false',
'--suite=SMOKE',
'--streamTimeout=60' ,
'--manageResources=false',
'--monitorJobs=true'
].join(' '))
}
shell('echo "*** RUN NEXMARK SQL IN BATCH MODE USING SPARK 3 RUNNER ***"')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":runners:spark:3"' +
' -Pnexmark.args="' +
[
commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
'--runner=SparkRunner',
'--queryLanguage=sql',
'--streaming=false',
'--suite=SMOKE',
'--streamTimeout=60' ,
'--streamTimeout=60',
'--manageResources=false',
'--monitorJobs=true'
].join(' '))
].join(' ') + '"')
}
shell('echo "*** RUN NEXMARK IN BATCH MODE USING SPARK 2 STRUCTURED STREAMING RUNNER ***"')
shell("echo \"*** RUN NEXMARK IN BATCH MODE USING SPARK ${CommonTestProperties.getSparkVersion()} STRUCTURED STREAMING RUNNER ***\"")
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":runners:spark:2"' +
switches("-Pnexmark.runner=\":runners:spark:${CommonTestProperties.getSparkVersion()}\"" +
' -Pnexmark.args="' +
[
commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
Expand All @@ -128,17 +91,17 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
'--suite=SMOKE',
// Skip query 3 (SparkStructuredStreamingRunner does not support State/Timers yet)
'--skipQueries=3',
'--streamTimeout=60' ,
'--streamTimeout=60',
'--manageResources=false',
'--monitorJobs=true'
].join(' '))
].join(' ') + '"')
}
shell('echo "*** RUN NEXMARK SQL IN BATCH MODE USING SPARK 2 STRUCTURED STREAMING RUNNER ***"')
shell("echo \"*** RUN NEXMARK SQL IN BATCH MODE USING SPARK ${CommonTestProperties.getSparkVersion()} STRUCTURED STREAMING RUNNER ***\"")
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":runners:spark:2"' +
switches("-Pnexmark.runner=\":runners:spark:${CommonTestProperties.getSparkVersion()}\"" +
' -Pnexmark.args="' +
[
commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
Expand All @@ -147,49 +110,10 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
'--queryLanguage=sql',
'--streaming=false',
'--suite=SMOKE',
'--streamTimeout=60' ,
'--manageResources=false',
'--monitorJobs=true'
].join(' '))
}
shell('echo "*** RUN NEXMARK IN BATCH MODE USING SPARK 3 STRUCTURED STREAMING RUNNER ***"')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":runners:spark:3"' +
' -Pnexmark.args="' +
[
commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
'--runner=SparkStructuredStreamingRunner',
'--streaming=false',
'--suite=SMOKE',
// Skip query 3 (SparkStructuredStreamingRunner does not support State/Timers yet)
'--skipQueries=3',
'--streamTimeout=60' ,
'--manageResources=false',
'--monitorJobs=true'
].join(' '))
}
shell('echo "*** RUN NEXMARK SQL IN BATCH MODE USING SPARK 3 STRUCTURED STREAMING RUNNER ***"')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":runners:spark:3"' +
' -Pnexmark.args="' +
[
commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
'--runner=SparkStructuredStreamingRunner',
'--queryLanguage=sql',
'--streaming=false',
'--suite=SMOKE',
'--streamTimeout=60' ,
'--streamTimeout=60',
'--manageResources=false',
'--monitorJobs=true'
].join(' '))
].join(' ') + '"')
}
}
}
Expand Down

0 comments on commit c2aa228

Please sign in to comment.