From df5e3b59ed1711fa519be43d62ef6266ab68e1e2 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Wed, 19 Oct 2022 18:33:45 +0200 Subject: [PATCH] Migrate examples and maven-archetypes (including Java Quickstart) to Spark 3 (addresses #23728) --- examples/java/build.gradle | 7 +---- examples/kotlin/build.gradle | 7 +---- release/build.gradle.kts | 2 +- runners/spark/3/build.gradle | 3 ++ runners/spark/spark_runner.gradle | 3 -- .../maven-archetypes/examples/build.gradle | 2 +- .../resources/archetype-resources/pom.xml | 28 +++---------------- .../gcp-bom-examples/build.gradle | 2 +- .../resources/archetype-resources/pom.xml | 24 ++-------------- 9 files changed, 15 insertions(+), 63 deletions(-) diff --git a/examples/java/build.gradle b/examples/java/build.gradle index 13b2518bf382..aa51dcfeae85 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -109,13 +109,8 @@ dependencies { } directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow") flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}") - // TODO: Make the netty version used configurable, we add netty-all 4.1.17.Final so it appears on the classpath - // before 4.1.8.Final defined by Apache Beam - sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final" - sparkRunnerPreCommit project(":runners:spark:2") + sparkRunnerPreCommit project(":runners:spark:3") sparkRunnerPreCommit project(":sdks:java:io:hadoop-file-system") - sparkRunnerPreCommit library.java.spark_streaming - sparkRunnerPreCommit library.java.spark_core } /* diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index 0aa3dc257b09..79a1248712d0 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -81,13 +81,8 @@ dependencies { } directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow") flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}") - // TODO: Make the netty version used configurable, we add netty-all 4.1.17.Final so it appears on the classpath - // before 4.1.8.Final defined by Apache Beam - sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final" - sparkRunnerPreCommit project(":runners:spark:2") + sparkRunnerPreCommit project(":runners:spark:3") sparkRunnerPreCommit project(":sdks:java:io:hadoop-file-system") - sparkRunnerPreCommit library.java.spark_streaming - sparkRunnerPreCommit library.java.spark_core } /* diff --git a/release/build.gradle.kts b/release/build.gradle.kts index 7de4ab3af61a..ce895af80f8b 100644 --- a/release/build.gradle.kts +++ b/release/build.gradle.kts @@ -38,7 +38,7 @@ task("runJavaExamplesValidationTask") { description = "Run the Beam quickstart across all Java runners" dependsOn(":runners:direct-java:runQuickstartJavaDirect") dependsOn(":runners:google-cloud-dataflow-java:runQuickstartJavaDataflow") - dependsOn(":runners:spark:2:runQuickstartJavaSpark") + dependsOn(":runners:spark:3:runQuickstartJavaSpark") dependsOn(":runners:flink:1.13:runQuickstartJavaFlinkLocal") dependsOn(":runners:direct-java:runMobileGamingJavaDirect") dependsOn(":runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow") diff --git a/runners/spark/3/build.gradle b/runners/spark/3/build.gradle index 3d59bd525c4b..494d367131b4 100644 --- a/runners/spark/3/build.gradle +++ b/runners/spark/3/build.gradle @@ -29,6 +29,9 @@ project.ext { // Load the main build script which contains all build logic. apply from: "$basePath/spark_runner.gradle" +// Generates runQuickstartJavaSpark task (can only support 1 version of Spark) +createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Spark') + // Additional supported Spark versions (used in compatibility tests) def sparkVersions = [ "330": "3.3.0", diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index 14a433162fb6..1869f9c21742 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -385,9 +385,6 @@ tasks.register("validatesRunner") { //dependsOn validatesStructuredStreamingRunnerBatch } -// Generates :runners:spark:*:runQuickstartJavaSpark task -createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Spark') - tasks.register("hadoopVersionsTest") { group = "Verification" dependsOn hadoopVersions.collect{k,v -> "hadoopVersion${k}Test"} diff --git a/sdks/java/maven-archetypes/examples/build.gradle b/sdks/java/maven-archetypes/examples/build.gradle index 148015f43898..6a034029f10e 100644 --- a/sdks/java/maven-archetypes/examples/build.gradle +++ b/sdks/java/maven-archetypes/examples/build.gradle @@ -36,7 +36,7 @@ processResources { 'libraries-bom.version': dependencies.create(project.library.java.google_cloud_platform_libraries_bom).getVersion(), 'pubsub.version': dependencies.create(project.library.java.google_api_services_pubsub).getVersion(), 'slf4j.version': dependencies.create(project.library.java.slf4j_api).getVersion(), - 'spark.version': dependencies.create(project.library.java.spark_core).getVersion(), + 'spark.version': dependencies.create(project.library.java.spark3_core).getVersion(), 'nemo.version': dependencies.create(project.library.java.nemo_compiler_frontend_beam).getVersion(), 'hadoop.version': dependencies.create(project.library.java.hadoop_client).getVersion(), 'mockito.version': dependencies.create(project.library.java.mockito_core).getVersion(), diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml index 50515b812078..5560ca93257e 100644 --- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml +++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml @@ -220,15 +220,11 @@ spark-runner - - - 4.1.17.Final - + org.apache.beam - beam-runners-spark + beam-runners-spark-3 ${beam.version} runtime @@ -246,7 +242,7 @@ org.apache.spark - spark-streaming_2.11 + spark-streaming_2.12 ${spark.version} runtime @@ -258,26 +254,10 @@ com.fasterxml.jackson.module - jackson-module-scala_2.11 + jackson-module-scala_2.12 ${jackson.version} runtime - - - org.apache.beam - beam-sdks-java-io-google-cloud-platform - ${beam.version} - - - io.grpc - grpc-netty - - - io.netty - netty-handler - - - diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle index 0e4f394170e5..af06bfc41d8e 100644 --- a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle +++ b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle @@ -35,7 +35,7 @@ processResources { 'junit.version': dependencies.create(project.library.java.junit).getVersion(), 'pubsub.version': dependencies.create(project.library.java.google_api_services_pubsub).getVersion(), 'slf4j.version': dependencies.create(project.library.java.slf4j_api).getVersion(), - 'spark.version': dependencies.create(project.library.java.spark_core).getVersion(), + 'spark.version': dependencies.create(project.library.java.spark3_core).getVersion(), 'nemo.version': dependencies.create(project.library.java.nemo_compiler_frontend_beam).getVersion(), 'hadoop.version': dependencies.create(project.library.java.hadoop_client).getVersion(), 'mockito.version': dependencies.create(project.library.java.mockito_core).getVersion(), diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml index 863a465f0fdc..c3fb0f26fcb2 100644 --- a/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml +++ b/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml @@ -216,13 +216,10 @@ spark-runner - - 4.1.17.Final - org.apache.beam - beam-runners-spark + beam-runners-spark-3 runtime @@ -238,7 +235,7 @@ org.apache.spark - spark-streaming_2.11 + spark-streaming_2.12 runtime @@ -249,25 +246,10 @@ com.fasterxml.jackson.module - jackson-module-scala_2.11 + jackson-module-scala_2.12 ${jackson.version} runtime - - - org.apache.beam - beam-sdks-java-io-google-cloud-platform - - - io.grpc - grpc-netty - - - io.netty - netty-handler - - -