From 696469fd2dc5b447e58cd22de090abc8d7399833 Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Thu, 19 Jan 2023 06:22:55 -0800 Subject: [PATCH] [#25070] Remove Kafka jars from Beam Java SDK Container (#25072) (#25074) This was added as a temporary work-around when the XLang expansion service did not support returning additional dependencies. This removes that dependency prevent conflicts and allowing the user to control which Kafka dependency version they want to use. For #25070 --- sdks/java/container/Dockerfile | 5 ----- sdks/java/container/boot.go | 2 -- sdks/java/container/build.gradle | 9 +-------- sdks/java/container/common.gradle | 2 -- 4 files changed, 1 insertion(+), 17 deletions(-) diff --git a/sdks/java/container/Dockerfile b/sdks/java/container/Dockerfile index b941aee453d4..5f25c1e65a9f 100644 --- a/sdks/java/container/Dockerfile +++ b/sdks/java/container/Dockerfile @@ -28,11 +28,6 @@ ADD target/log4j-over-slf4j.jar /opt/apache/beam/jars/ ADD target/log4j-to-slf4j.jar /opt/apache/beam/jars/ ADD target/beam-sdks-java-harness.jar /opt/apache/beam/jars/ -# Required to run cross-language pipelines with KafkaIO -# TODO May be removed once custom environments are supported -ADD target/beam-sdks-java-io-kafka.jar /opt/apache/beam/jars/ -ADD target/kafka-clients.jar /opt/apache/beam/jars/ - # Required to use jamm as a javaagent to get accurate object size measuring # COPY fails if file is not found, so use a wildcard for open-module-agent.jar # since it is only included in Java 9+ containers diff --git a/sdks/java/container/boot.go b/sdks/java/container/boot.go index 63dd1176d370..a5a42d4fb1e1 100644 --- a/sdks/java/container/boot.go +++ b/sdks/java/container/boot.go @@ -141,8 +141,6 @@ func main() { filepath.Join(jarsDir, "log4j-over-slf4j.jar"), filepath.Join(jarsDir, "log4j-to-slf4j.jar"), filepath.Join(jarsDir, "beam-sdks-java-harness.jar"), - filepath.Join(jarsDir, "beam-sdks-java-io-kafka.jar"), - filepath.Join(jarsDir, "kafka-clients.jar"), } var hasWorkerExperiment = strings.Contains(options, "use_staged_dataflow_worker_jar") diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle index 98a626595603..133ef382f64e 100644 --- a/sdks/java/container/build.gradle +++ b/sdks/java/container/build.gradle @@ -25,7 +25,6 @@ applyGoNature() applyPythonNature() evaluationDependsOn(":sdks:java:harness") -evaluationDependsOn(":sdks:java:io:kafka") evaluationDependsOn(":sdks:java:io:jdbc") description = "Apache Beam :: SDKs :: Java :: Container" @@ -42,10 +41,6 @@ dependencies { dockerDependency library.java.log4j_over_slf4j dockerDependency library.java.log4j2_to_slf4j dockerDependency project(path: ":sdks:java:harness", configuration: "shadow") - // For executing KafkaIO, e.g. as an external transform - dockerDependency project(":sdks:java:io:kafka") - // This dependency is set to 'provided' scope in :sdks:java:io:kafka - dockerDependency library.java.kafka_clients dockerDependency library.java.jamm } @@ -58,9 +53,7 @@ goBuild { import com.github.jk1.license.render.* licenseReport { projects = [project, - project.rootProject.findProject(':sdks:java:harness'), - project.rootProject.findProject(':sdks:java:io:kafka'), - project.rootProject.findProject(':sdks:java:io:jdbc')] + project.rootProject.findProject(':sdks:java:harness')] excludeOwnGroup = true excludeGroups = ["beam.*"] // project dependencies do not match their maven coords configurations = ALL diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index 1ec0da0098b5..d3f21096a620 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -55,8 +55,6 @@ task copyDockerfileDependencies(type: Copy) { rename 'beam-sdks-java-container-agent.*.jar', 'open-module-agent.jar' } rename 'beam-sdks-java-harness-.*.jar', 'beam-sdks-java-harness.jar' - rename 'beam-sdks-java-io-kafka.*.jar', 'beam-sdks-java-io-kafka.jar' - rename 'kafka-clients.*.jar', 'kafka-clients.jar' rename 'jamm.*.jar', 'jamm.jar' setDuplicatesStrategy(DuplicatesStrategy.INCLUDE)