Skip to content

Commit 3134f11

Browse files
koeningertdas
koeninger
authored and
tdas
committed
[SPARK-12177][STREAMING][KAFKA] Update KafkaDStreams to new Kafka 0.10 Consumer API
## What changes were proposed in this pull request? New Kafka consumer api for the released 0.10 version of Kafka ## How was this patch tested? Unit tests, manual tests Author: cody koeninger <cody@koeninger.org> Closes #11863 from koeninger/kafka-0.9. (cherry picked from commit dedbcee) Signed-off-by: Tathagata Das <tathagata.das1565@gmail.com>
1 parent a548523 commit 3134f11

File tree

22 files changed

+3359
-6
lines changed

22 files changed

+3359
-6
lines changed

‎external/kafka-0-10-assembly/pom.xml‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.spark</groupId>
23+
<artifactId>spark-parent_2.11</artifactId>
24+
<version>2.0.0-SNAPSHOT</version>
25+
<relativePath>../../pom.xml</relativePath>
26+
</parent>
27+
28+
<groupId>org.apache.spark</groupId>
29+
<artifactId>spark-streaming-kafka-0-10-assembly_2.11</artifactId>
30+
<packaging>jar</packaging>
31+
<name>Spark Integration for Kafka 0.10 Assembly</name>
32+
<url>http://spark.apache.org/</url>
33+
34+
<properties>
35+
<sbt.project.name>streaming-kafka-0-10-assembly</sbt.project.name>
36+
</properties>
37+
38+
<dependencies>
39+
<dependency>
40+
<groupId>org.apache.spark</groupId>
41+
<artifactId>spark-streaming-kafka-0-10_${scala.binary.version}</artifactId>
42+
<version>${project.version}</version>
43+
</dependency>
44+
<dependency>
45+
<groupId>org.apache.spark</groupId>
46+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
47+
<version>${project.version}</version>
48+
<scope>provided</scope>
49+
</dependency>
50+
<!--
51+
Demote already included in the Spark assembly.
52+
-->
53+
<dependency>
54+
<groupId>commons-codec</groupId>
55+
<artifactId>commons-codec</artifactId>
56+
<scope>provided</scope>
57+
</dependency>
58+
<dependency>
59+
<groupId>commons-lang</groupId>
60+
<artifactId>commons-lang</artifactId>
61+
<scope>provided</scope>
62+
</dependency>
63+
<dependency>
64+
<groupId>com.google.protobuf</groupId>
65+
<artifactId>protobuf-java</artifactId>
66+
<scope>provided</scope>
67+
</dependency>
68+
<dependency>
69+
<groupId>net.jpountz.lz4</groupId>
70+
<artifactId>lz4</artifactId>
71+
<scope>provided</scope>
72+
</dependency>
73+
<dependency>
74+
<groupId>org.apache.hadoop</groupId>
75+
<artifactId>hadoop-client</artifactId>
76+
<scope>provided</scope>
77+
</dependency>
78+
<dependency>
79+
<groupId>org.apache.avro</groupId>
80+
<artifactId>avro-mapred</artifactId>
81+
<classifier>${avro.mapred.classifier}</classifier>
82+
<scope>provided</scope>
83+
</dependency>
84+
<dependency>
85+
<groupId>org.apache.curator</groupId>
86+
<artifactId>curator-recipes</artifactId>
87+
<scope>provided</scope>
88+
</dependency>
89+
<dependency>
90+
<groupId>org.apache.zookeeper</groupId>
91+
<artifactId>zookeeper</artifactId>
92+
<scope>provided</scope>
93+
</dependency>
94+
<dependency>
95+
<groupId>log4j</groupId>
96+
<artifactId>log4j</artifactId>
97+
<scope>provided</scope>
98+
</dependency>
99+
<dependency>
100+
<groupId>net.java.dev.jets3t</groupId>
101+
<artifactId>jets3t</artifactId>
102+
<scope>provided</scope>
103+
</dependency>
104+
<dependency>
105+
<groupId>org.scala-lang</groupId>
106+
<artifactId>scala-library</artifactId>
107+
<scope>provided</scope>
108+
</dependency>
109+
<dependency>
110+
<groupId>org.slf4j</groupId>
111+
<artifactId>slf4j-api</artifactId>
112+
<scope>provided</scope>
113+
</dependency>
114+
<dependency>
115+
<groupId>org.slf4j</groupId>
116+
<artifactId>slf4j-log4j12</artifactId>
117+
<scope>provided</scope>
118+
</dependency>
119+
<dependency>
120+
<groupId>org.xerial.snappy</groupId>
121+
<artifactId>snappy-java</artifactId>
122+
<scope>provided</scope>
123+
</dependency>
124+
</dependencies>
125+
126+
<build>
127+
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
128+
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
129+
<plugins>
130+
<plugin>
131+
<groupId>org.apache.maven.plugins</groupId>
132+
<artifactId>maven-shade-plugin</artifactId>
133+
<configuration>
134+
<shadedArtifactAttached>false</shadedArtifactAttached>
135+
<artifactSet>
136+
<includes>
137+
<include>*:*</include>
138+
</includes>
139+
</artifactSet>
140+
<filters>
141+
<filter>
142+
<artifact>*:*</artifact>
143+
<excludes>
144+
<exclude>META-INF/*.SF</exclude>
145+
<exclude>META-INF/*.DSA</exclude>
146+
<exclude>META-INF/*.RSA</exclude>
147+
</excludes>
148+
</filter>
149+
</filters>
150+
</configuration>
151+
<executions>
152+
<execution>
153+
<phase>package</phase>
154+
<goals>
155+
<goal>shade</goal>
156+
</goals>
157+
<configuration>
158+
<transformers>
159+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
160+
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
161+
<resource>reference.conf</resource>
162+
</transformer>
163+
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
164+
<resource>log4j.properties</resource>
165+
</transformer>
166+
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
167+
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
168+
</transformers>
169+
</configuration>
170+
</execution>
171+
</executions>
172+
</plugin>
173+
</plugins>
174+
</build>
175+
</project>
176+

‎external/kafka-0-10/pom.xml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.spark</groupId>
23+
<artifactId>spark-parent_2.11</artifactId>
24+
<version>2.0.0-SNAPSHOT</version>
25+
<relativePath>../../pom.xml</relativePath>
26+
</parent>
27+
28+
<groupId>org.apache.spark</groupId>
29+
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
30+
<properties>
31+
<sbt.project.name>streaming-kafka-0-10</sbt.project.name>
32+
</properties>
33+
<packaging>jar</packaging>
34+
<name>Spark Integration for Kafka 0.10</name>
35+
<url>http://spark.apache.org/</url>
36+
37+
<dependencies>
38+
<dependency>
39+
<groupId>org.apache.spark</groupId>
40+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
41+
<version>${project.version}</version>
42+
<scope>provided</scope>
43+
</dependency>
44+
<dependency>
45+
<groupId>org.apache.spark</groupId>
46+
<artifactId>spark-core_${scala.binary.version}</artifactId>
47+
<version>${project.version}</version>
48+
<type>test-jar</type>
49+
<scope>test</scope>
50+
</dependency>
51+
<dependency>
52+
<groupId>org.apache.kafka</groupId>
53+
<artifactId>kafka_${scala.binary.version}</artifactId>
54+
<version>0.10.0.0</version>
55+
<exclusions>
56+
<exclusion>
57+
<groupId>com.sun.jmx</groupId>
58+
<artifactId>jmxri</artifactId>
59+
</exclusion>
60+
<exclusion>
61+
<groupId>com.sun.jdmk</groupId>
62+
<artifactId>jmxtools</artifactId>
63+
</exclusion>
64+
<exclusion>
65+
<groupId>net.sf.jopt-simple</groupId>
66+
<artifactId>jopt-simple</artifactId>
67+
</exclusion>
68+
<exclusion>
69+
<groupId>org.slf4j</groupId>
70+
<artifactId>slf4j-simple</artifactId>
71+
</exclusion>
72+
<exclusion>
73+
<groupId>org.apache.zookeeper</groupId>
74+
<artifactId>zookeeper</artifactId>
75+
</exclusion>
76+
</exclusions>
77+
</dependency>
78+
<dependency>
79+
<groupId>net.sf.jopt-simple</groupId>
80+
<artifactId>jopt-simple</artifactId>
81+
<version>3.2</version>
82+
<scope>test</scope>
83+
</dependency>
84+
<dependency>
85+
<groupId>org.scalacheck</groupId>
86+
<artifactId>scalacheck_${scala.binary.version}</artifactId>
87+
<scope>test</scope>
88+
</dependency>
89+
<dependency>
90+
<groupId>org.apache.spark</groupId>
91+
<artifactId>spark-tags_${scala.binary.version}</artifactId>
92+
</dependency>
93+
</dependencies>
94+
<build>
95+
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
96+
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
97+
</build>
98+
</project>

0 commit comments

Comments
 (0)