[Spark runner] Removal of Spark 2 runner support (closes #25259)

.test-infra/jenkins/job_PostCommit_Java_Examples_Spark.groovy

@@ -36,7 +36,6 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Spark',
  steps {
  gradle {
  rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:spark:2:examplesIntegrationTest')
  tasks(':runners:spark:3:examplesIntegrationTest')
  commonJobProperties.setGradleSwitches(delegate)
  }

.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark_Batch.groovy

@@ -36,9 +36,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Spark_Batch',
  steps {
  gradle {
  rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:spark:2:job-server:validatesPortableRunnerBatch')
  tasks(':runners:spark:3:job-server:validatesPortableRunnerBatch')
- tasks(':runners:spark:2:job-server:validatesPortableRunnerDocker')
  tasks(':runners:spark:3:job-server:validatesPortableRunnerDocker')
  commonJobProperties.setGradleSwitches(delegate)
  }

.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy

@@ -37,7 +37,6 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Spark',
  steps {
  gradle {
  rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:spark:2:validatesRunner')
  tasks(':runners:spark:3:validatesRunner')
  commonJobProperties.setGradleSwitches(delegate)
  }

.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.groovy

@@ -36,7 +36,6 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_SparkSt
  steps {
  gradle {
  rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:spark:2:validatesStructuredStreamingRunnerBatch')
  tasks(':runners:spark:3:validatesStructuredStreamingRunnerBatch')
  commonJobProperties.setGradleSwitches(delegate)
  }

CHANGES.md

@@ -68,7 +68,7 @@
 
 ## Breaking Changes
 
-* X behavior was changed ([#X](https://github.com/apache/beam/issues/X)).
+* The deprecated SparkRunner for Spark 2 (see [2.41.0](#2410---2022-08-23)) was finally removed ([#25263](https://github.com/apache/beam/pull/25263)).
 
 ## Deprecations
 

build.gradle.kts

@@ -254,8 +254,6 @@ tasks.register("javaPreCommit") {
  dependsOn(":runners:portability:java:build")
  dependsOn(":runners:samza:build")
  dependsOn(":runners:samza:job-server:build")
- dependsOn(":runners:spark:2:build")
- dependsOn(":runners:spark:2:job-server:build")
  dependsOn(":runners:spark:3:build")
  dependsOn(":runners:spark:3:job-server:build")
  dependsOn(":runners:twister2:build")
@@ -340,7 +338,6 @@ tasks.register("javaPostCommitSickbay") {
  dependsOn(":runners:flink:1.13:validatesRunnerSickbay")
  dependsOn(":runners:flink:1.14:validatesRunnerSickbay")
  dependsOn(":runners:flink:1.15:validatesRunnerSickbay")
- dependsOn(":runners:spark:2:job-server:validatesRunnerSickbay")
  dependsOn(":runners:spark:3:job-server:validatesRunnerSickbay")
  dependsOn(":runners:direct-java:validatesRunnerSickbay")
  dependsOn(":runners:portability:java:validatesRunnerSickbay")
@@ -353,7 +350,6 @@ tasks.register("javaHadoopVersionsTest") {
  dependsOn(":sdks:java:io:hcatalog:hadoopVersionsTest")
  dependsOn(":sdks:java:io:parquet:hadoopVersionsTest")
  dependsOn(":sdks:java:extensions:sorter:hadoopVersionsTest")
- dependsOn(":runners:spark:2:hadoopVersionsTest")
  dependsOn(":runners:spark:3:hadoopVersionsTest")
 }
 
@@ -548,7 +544,6 @@ tasks.register("typescriptPreCommit") {
 }
 
 tasks.register("pushAllDockerImages") {
- dependsOn(":runners:spark:2:job-server:container:dockerPush")
  dependsOn(":runners:spark:3:job-server:container:dockerPush")
  dependsOn(":sdks:java:container:pushAll")
  dependsOn(":sdks:python:container:pushAll")

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -505,7 +505,6 @@ class BeamModulePlugin implements Plugin<Project> {
  def jmh_version = "1.34"
 
  // Export Spark versions, so they are defined in a single place only
- project.ext.spark2_version = spark2_version
  project.ext.spark3_version = spark3_version
 
  // A map of maps containing common libraries used per language. To use:
@@ -730,8 +729,6 @@ class BeamModulePlugin implements Plugin<Project> {
  slf4j_jcl : "org.slf4j:slf4j-jcl:$slf4j_version",
  snappy_java : "org.xerial.snappy:snappy-java:1.1.8.4",
  spark_core : "org.apache.spark:spark-core_2.11:$spark2_version",
- spark_network_common : "org.apache.spark:spark-network-common_2.11:$spark2_version",
- spark_sql : "org.apache.spark:spark-sql_2.11:$spark2_version",
  spark_streaming : "org.apache.spark:spark-streaming_2.11:$spark2_version",
  spark3_core : "org.apache.spark:spark-core_2.12:$spark3_version",
  spark3_network_common : "org.apache.spark:spark-network-common_2.12:$spark3_version",