STORM-2554: Trident Kafka Spout Refactoring to Include Manual Partition Assignment

examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/KafkaProducerTopology.java

@@ -54,7 +54,7 @@ public static StormTopology newTopology(String brokerUrl, String topicName) {
     /**
      * @return the Storm config for the topology that publishes sentences to kafka using a kafka bolt.
      */
-    private static Properties newProps(final String brokerUrl, final String topicName) {
+    static Properties newProps(final String brokerUrl, final String topicName) {
         return new Properties() {{
             put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerUrl);
             put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");

external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java

@@ -78,7 +78,11 @@ public static enum FirstPollOffsetStrategy {
         LATEST,
         UNCOMMITTED_EARLIEST,
         UNCOMMITTED_LATEST }
-
+
+    public static Builder<String, String> builder(String bootstrapServers, Subscription subscription) {
+        return new Builder<>(bootstrapServers, StringDeserializer.class, StringDeserializer.class, subscription);
+    }
+
     public static Builder<String, String> builder(String bootstrapServers, String ... topics) {
         return new Builder<>(bootstrapServers, StringDeserializer.class, StringDeserializer.class, topics);
     }

external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/ManualPartitionPatternSubscription.java

@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *   or more contributor license agreements.  See the NOTICE file
- *   distributed with this work for additional information
- *   regarding copyright ownership.  The ASF licenses this file
- *   to you under the Apache License, Version 2.0 (the
- *   "License"); you may not use this file except in compliance
- *   with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *   Unless required by applicable law or agreed to in writing, software
- *   distributed under the License is distributed on an "AS IS" BASIS,
- *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *   See the License for the specific language governing permissions and
- *   limitations under the License.
- */
-
-package org.apache.storm.kafka.spout;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.storm.task.TopologyContext;
-
-public class ManualPartitionPatternSubscription extends PatternSubscription {
-    private static final long serialVersionUID = 5633018073527583826L;
-    private final ManualPartitioner parter;
-    private Set<TopicPartition> currentAssignment = null;
-    private KafkaConsumer<?, ?> consumer = null;
-    private ConsumerRebalanceListener listener = null;
-    private TopologyContext context = null;
-
-    public ManualPartitionPatternSubscription(ManualPartitioner parter, Pattern pattern) {
-        super(pattern);
-        this.parter = parter;
-    }
-
-    @Override
-    public <K, V> void subscribe(KafkaConsumer<K, V> consumer, ConsumerRebalanceListener listener, TopologyContext context) {
-        this.consumer = consumer;
-        this.listener = listener;
-        this.context = context;
-        refreshAssignment();
-    }
-
-    @Override
-    public void refreshAssignment() {
-        List<TopicPartition> allPartitions = new ArrayList<>();
-        for (Map.Entry<String, List<PartitionInfo>> entry: consumer.listTopics().entrySet()) {
-            if (pattern.matcher(entry.getKey()).matches()) {
-                for (PartitionInfo partitionInfo: entry.getValue()) {
-                    allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
-                }
-            }
-        }
-        Collections.sort(allPartitions, TopicPartitionComparator.INSTANCE);
-        Set<TopicPartition> newAssignment = new HashSet<>(parter.partition(allPartitions, context));
-        if (!newAssignment.equals(currentAssignment)) {
-            if (currentAssignment != null) {
-                listener.onPartitionsRevoked(currentAssignment);
-                listener.onPartitionsAssigned(newAssignment);
-            }
-            currentAssignment = newAssignment;
-            consumer.assign(currentAssignment);
-        }
-    }
-}

external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/ManualPartitionNamedSubscription.java → external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/ManualPartitionSubscription.java

@@ -18,36 +18,29 @@
 
 package org.apache.storm.kafka.spout;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.storm.task.TopologyContext;
 
-public class ManualPartitionNamedSubscription extends NamedSubscription {
+public class ManualPartitionSubscription extends Subscription {
     private static final long serialVersionUID = 5633018073527583826L;
     private final ManualPartitioner partitioner;
+    private final TopicFilter partitionFilter;
     private Set<TopicPartition> currentAssignment = null;
     private KafkaConsumer<?, ?> consumer = null;
     private ConsumerRebalanceListener listener = null;
     private TopologyContext context = null;
 
-    public ManualPartitionNamedSubscription(ManualPartitioner parter, Collection<String> topics) {
-        super(topics);
+    public ManualPartitionSubscription(ManualPartitioner parter, TopicFilter partitionFilter) {
+        this.partitionFilter = partitionFilter;
         this.partitioner = parter;
     }
 
-    public ManualPartitionNamedSubscription(ManualPartitioner parter, String ... topics) {
-        this(parter, Arrays.asList(topics));
-    }
-
     @Override
     public <K, V> void subscribe(KafkaConsumer<K, V> consumer, ConsumerRebalanceListener listener, TopologyContext context) {
         this.consumer = consumer;
@@ -58,21 +51,21 @@ public <K, V> void subscribe(KafkaConsumer<K, V> consumer, ConsumerRebalanceList
 
     @Override
     public void refreshAssignment() {
-        List<TopicPartition> allPartitions = new ArrayList<>();
-        for (String topic : topics) {
-            for (PartitionInfo partitionInfo: consumer.partitionsFor(topic)) {
-                allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
-            }
-        }
+        final List<TopicPartition> allPartitions = partitionFilter.getFilteredTopicPartitions(consumer);
         Collections.sort(allPartitions, TopicPartitionComparator.INSTANCE);
-        Set<TopicPartition> newAssignment = new HashSet<>(partitioner.partition(allPartitions, context));
+        final Set<TopicPartition> newAssignment = new HashSet<>(partitioner.partition(allPartitions, context));
         if (!newAssignment.equals(currentAssignment)) {
+            consumer.assign(newAssignment);
             if (currentAssignment != null) {
                 listener.onPartitionsRevoked(currentAssignment);
-                listener.onPartitionsAssigned(newAssignment);
             }
             currentAssignment = newAssignment;
-            consumer.assign(currentAssignment);
+            listener.onPartitionsAssigned(newAssignment);
         }
     }
+
+    @Override
+    public String getTopicsString() {
+        return partitionFilter.getTopicsString();
+    }
 }

external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/ManualPartitioner.java

@@ -18,6 +18,7 @@
 
 package org.apache.storm.kafka.spout;
 
+import java.io.Serializable;
 import java.util.List;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.storm.task.TopologyContext;
@@ -29,7 +30,7 @@
  * number of spouts to avoid missing partitions or double assigning partitions.
  */
 @FunctionalInterface
-public interface ManualPartitioner {
+public interface ManualPartitioner extends Serializable {
     /**
      * Get the partitions for this assignment
      * @param allPartitions all of the partitions that the set of spouts want to subscribe to, in a strict ordering

external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.kafka.spout;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+
+/**
+ * Filter that returns all partitions for the specified topics.
+ */
+public class NamedTopicFilter implements TopicFilter {
+
+    private final Set<String> topics;
+
+    /**
+     * Create filter based on a set of topic names.
+     * @param topics The topic names the filter will pass.
+     */
+    public NamedTopicFilter(Set<String> topics) {
+        this.topics = Collections.unmodifiableSet(topics);
+    }
+
+    /**
+     * Convenience constructor.
+     * @param topics The topic names the filter will pass.
+     */
+    public NamedTopicFilter(String... topics) {
+        this(new HashSet<>(Arrays.asList(topics)));
+    }
+
+    @Override
+    public List<TopicPartition> getFilteredTopicPartitions(KafkaConsumer<?, ?> consumer) {
+        final List<TopicPartition> allPartitions = new ArrayList<>();
+        for (String topic : topics) {
+            for (PartitionInfo partitionInfo: consumer.partitionsFor(topic)) {
+                allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
+            }
+        }
+        return allPartitions;
+    }
+
+    @Override
+    public String getTopicsString() {
+        return String.join(",", topics);
+    }
+}

external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/PatternTopicFilter.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.kafka.spout;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+
+/**
+ * Filter that returns all partitions for topics matching the given {@link Pattern}.
+ */
+public class PatternTopicFilter implements TopicFilter {
+
+    private final Pattern pattern;
+    private Set<String> topics;
+
+    /**
+     * Creates filter based on a Pattern. Only topic names matching the Pattern are passed by the filter.
+     * @param pattern The Pattern to use.
+     */
+    public PatternTopicFilter(Pattern pattern) {
+        this.pattern = pattern;
+    }
+
+    @Override
+    public List<TopicPartition> getFilteredTopicPartitions(KafkaConsumer<?, ?> consumer) {
+        topics = new HashSet<>();
+        List<TopicPartition> allPartitions = new ArrayList<>();
+        for (Map.Entry<String, List<PartitionInfo>> entry: consumer.listTopics().entrySet()) {
+            if (pattern.matcher(entry.getKey()).matches()) {
+                for (PartitionInfo partitionInfo: entry.getValue()) {
+                    allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
+                    topics.add(partitionInfo.topic());
+                }
+            }
+        }
+        return allPartitions;
+    }
+
+    @Override
+    public String getTopicsString() {
+        return String.join(",", topics);
+    }
+
+    public String getTopicsPattern() {
+        return pattern.pattern();
+    }
+}

external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/TopicFilter.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.kafka.spout;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.TopicPartition;
+
+public interface TopicFilter extends Serializable {
+
+    /**
+     * Get the Kafka TopicPartitions passed by this filter. 
+     * @param consumer The Kafka consumer to use to read the list of existing partitions
+     * @return The Kafka partitions passed by this filter.
+     */
+    List<TopicPartition> getFilteredTopicPartitions(KafkaConsumer<?, ?> consumer);
+
+    /**
+     * Gets a human readable representation of the topics 
+     * @return a string representing the subscribed topics.
+     */
+    String getTopicsString();
+
+}