[#25070] Remove Kafka jars from Beam Java SDK Container (#25072)

This was added as a temporary work-around when the XLang expansion service did not support returning additional dependencies. This removes that dependency prevent conflicts and allowing the user to control which Kafka dependency version they want to use. For #25070
apache · Jan 19, 2023 · 13a740b · 13a740b
1 parent 6daa6f4
commit 13a740b
Show file tree

Hide file tree

Showing 4 changed files with 1 addition and 17 deletions.
diff --git a/sdks/java/container/Dockerfile b/sdks/java/container/Dockerfile
@@ -28,11 +28,6 @@ ADD target/log4j-over-slf4j.jar /opt/apache/beam/jars/
 ADD target/log4j-to-slf4j.jar /opt/apache/beam/jars/
 ADD target/beam-sdks-java-harness.jar /opt/apache/beam/jars/
 
-# Required to run cross-language pipelines with KafkaIO
-# TODO May be removed once custom environments are supported
-ADD target/beam-sdks-java-io-kafka.jar /opt/apache/beam/jars/
-ADD target/kafka-clients.jar /opt/apache/beam/jars/
-
 # Required to use jamm as a javaagent to get accurate object size measuring
 # COPY fails if file is not found, so use a wildcard for open-module-agent.jar
 # since it is only included in Java 9+ containers

diff --git a/sdks/java/container/boot.go b/sdks/java/container/boot.go
@@ -141,8 +141,6 @@ func main() {
  filepath.Join(jarsDir, "log4j-over-slf4j.jar"),
  filepath.Join(jarsDir, "log4j-to-slf4j.jar"),
  filepath.Join(jarsDir, "beam-sdks-java-harness.jar"),
- filepath.Join(jarsDir, "beam-sdks-java-io-kafka.jar"),
- filepath.Join(jarsDir, "kafka-clients.jar"),
  }
 
  var hasWorkerExperiment = strings.Contains(options, "use_staged_dataflow_worker_jar")

diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle
@@ -25,7 +25,6 @@ applyGoNature()
 applyPythonNature()
 
 evaluationDependsOn(":sdks:java:harness")
-evaluationDependsOn(":sdks:java:io:kafka")
 evaluationDependsOn(":sdks:java:io:jdbc")
 
 description = "Apache Beam :: SDKs :: Java :: Container"
@@ -42,10 +41,6 @@ dependencies {
  dockerDependency library.java.log4j_over_slf4j
  dockerDependency library.java.log4j2_to_slf4j
  dockerDependency project(path: ":sdks:java:harness", configuration: "shadow")
- // For executing KafkaIO, e.g. as an external transform
- dockerDependency project(":sdks:java:io:kafka")
- // This dependency is set to 'provided' scope in :sdks:java:io:kafka
- dockerDependency library.java.kafka_clients
  dockerDependency library.java.jamm
 }
 
@@ -58,9 +53,7 @@ goBuild {
 import com.github.jk1.license.render.*
 licenseReport {
  projects = [project,
- project.rootProject.findProject(':sdks:java:harness'),
- project.rootProject.findProject(':sdks:java:io:kafka'),
- project.rootProject.findProject(':sdks:java:io:jdbc')]
+ project.rootProject.findProject(':sdks:java:harness')]
  excludeOwnGroup = true
  excludeGroups = ["beam.*"] // project dependencies do not match their maven coords
  configurations = ALL

diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle
@@ -55,8 +55,6 @@ task copyDockerfileDependencies(type: Copy) {
  rename 'beam-sdks-java-container-agent.*.jar', 'open-module-agent.jar'
  }
  rename 'beam-sdks-java-harness-.*.jar', 'beam-sdks-java-harness.jar'
- rename 'beam-sdks-java-io-kafka.*.jar', 'beam-sdks-java-io-kafka.jar'
- rename 'kafka-clients.*.jar', 'kafka-clients.jar'
  rename 'jamm.*.jar', 'jamm.jar'
 
  setDuplicatesStrategy(DuplicatesStrategy.INCLUDE)