Merge pull request apache#186 from ni-ze/snapshot-1.0.4

[ISSUE apache#187]Merge from Snapshot 1.0.4
ni-ze · Jul 2, 2022 · 2bd62ca · 2bd62ca
2 parents b285cac + b905345
commit 2bd62ca
Show file tree

Hide file tree

Showing 211 changed files with 9,648 additions and 2,380 deletions.
diff --git a/pom.xml b/pom.xml
@@ -15,7 +15,9 @@
  See the License for the specific language governing permissions and
  limitations under the License.
  -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
 
  <parent>
@@ -31,56 +33,6 @@
  <packaging>pom</packaging>
  <url>https://rocketmq.apache.org/</url>
 
-
- <scm>
- <url>git@github.com:apache/rocketmq-streams.git</url>
- <connection>scm:git:git@github.com:apache/rocketmq-streams.git</connection>
- <developerConnection>scm:git:git@github.com:apache/rocketmq-streams.git</developerConnection>
- <tag>1.0.0-preview</tag>
- </scm>
-
- <mailingLists>
- <mailingList>
- <name>Development List</name>
- <subscribe>dev-subscribe@rocketmq.apache.org</subscribe>
- <unsubscribe>dev-unsubscribe@rocketmq.apache.org</unsubscribe>
- <post>dev@rocketmq.apache.org</post>
- </mailingList>
- <mailingList>
- <name>User List</name>
- <subscribe>users-subscribe@rocketmq.apache.org</subscribe>
- <unsubscribe>users-unsubscribe@rocketmq.apache.org</unsubscribe>
- <post>users@rocketmq.apache.org</post>
- </mailingList>
- <mailingList>
- <name>Commits List</name>
- <subscribe>commits-subscribe@rocketmq.apache.org</subscribe>
- <unsubscribe>commits-unsubscribe@rocketmq.apache.org</unsubscribe>
- <post>commits@rocketmq.apache.org</post>
- </mailingList>
- </mailingLists>
-
- <developers>
- <developer>
- <id>Apache RocketMQ</id>
- <name>Apache RocketMQ of ASF</name>
- <url>https://rocketmq.apache.org/</url>
- </developer>
- </developers>
-
- <organization>
- <name>Apache Software Foundation</name>
- <url>http://www.apache.org</url>
- </organization>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
-
  <modules>
  <module>rocketmq-streams-commons</module>
  <module>rocketmq-streams-transport-minio</module>
@@ -112,13 +64,13 @@
  <java.version>1.8</java.version>
  <java.encoding>UTF-8</java.encoding>
  <project.build.sourceEncoding>${java.encoding}</project.build.sourceEncoding>
- <log4j.version>1.2.17</log4j.version>
  <commons-logging.version>1.1</commons-logging.version>
- <spring.version>3.2.13.RELEASE</spring.version>
+ <spring.version>5.1.14.RELEASE</spring.version>
  <auto-service.version>1.0-rc5</auto-service.version>
  <mysql-connector.version>5.1.40</mysql-connector.version>
  <fastjson.version>1.2.83</fastjson.version>
  <quartz.version>2.3.2</quartz.version>
+
  <httpclient.version>4.5.13</httpclient.version>
  <commons-io.version>2.7</commons-io.version>
  <junit.version>4.13.1</junit.version>
@@ -140,8 +92,10 @@
  <commons-lang.version>2.5</commons-lang.version>
  <elasticsearch.version>7.4.0</elasticsearch.version>
  <kafka.version>1.1.0</kafka.version>
- <maven.javadoc.skip>true</maven.javadoc.skip>
  <paho.version>1.2.2</paho.version>
+ <kryo.version>5.3.0</kryo.version>
+ <fst.version>2.56</fst.version>
+ <slf4j-log4j12.version>1.7.36</slf4j-log4j12.version>
  </properties>
 
 
@@ -167,10 +121,7 @@
  <exclude>build_without_test.sh</exclude>
  <exclude>NOTICE</exclude>
  <exclude>LICENSE</exclude>
- <exclude>README.md</exclude>
- <exclude>README-chinese.md</exclude>
- <exclude>QUICKSTART.md</exclude>
- <exclude>quick_start.md</exclude>
+ <exclude>*.md</exclude>
  <exclude>.github/**</exclude>
  <exclude>*/target/**</exclude>
  <exclude>*/*.iml</exclude>
@@ -202,18 +153,6 @@
  <artifactId>versions-maven-plugin</artifactId>
  <version>2.2</version>
  </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>cobertura-maven-plugin</artifactId>
- <version>2.7</version>
- <configuration>
- <formats>
- <format>html</format>
- <format>xml</format>
- </formats>
- <check />
- </configuration>
- </plugin>
  <plugin>
  <groupId>org.jacoco</groupId>
  <artifactId>jacoco-maven-plugin</artifactId>
@@ -260,11 +199,6 @@
  <artifactId>rocketmq-streams-commons</artifactId>
  <version>${project.version}</version>
  </dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-channel-es</artifactId>
- <version>${project.version}</version>
- </dependency>
  <dependency>
  <groupId>org.apache.rocketmq</groupId>
  <artifactId>rocketmq-streams-channel-mqtt</artifactId>
@@ -305,11 +239,6 @@
  <artifactId>rocketmq-streams-db-operator</artifactId>
  <version>${project.version}</version>
  </dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-dim</artifactId>
- <version>${project.version}</version>
- </dependency>
  <dependency>
  <groupId>org.apache.rocketmq</groupId>
  <artifactId>rocketmq-streams-filter</artifactId>
@@ -331,11 +260,6 @@
  </exclusion>
  </exclusions>
  </dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-script-python</artifactId>
- <version>${project.version}</version>
- </dependency>
  <dependency>
  <groupId>org.apache.rocketmq</groupId>
  <artifactId>rocketmq-streams-serviceloader</artifactId>
@@ -366,7 +290,6 @@
  <artifactId>rocketmq-streams-channel-rocketmq</artifactId>
  <version>${project.version}</version>
  </dependency>
-
  <dependency>
  <groupId>org.apache.rocketmq</groupId>
  <artifactId>rocketmq-streams-examples</artifactId>
@@ -398,12 +321,6 @@
  <version>${rocketmq.version}</version>
  </dependency>
 
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-tools</artifactId>
- <version>${rocketmq.version}</version>
- </dependency>
-
  <!-- ================================================= -->
  <!-- tool library -->
  <!-- ================================================= -->
@@ -437,9 +354,9 @@
  </dependency>
 
  <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j-log4j12.version}</version>
  </dependency>
 
  <dependency>
@@ -586,6 +503,12 @@
  <artifactId>elasticsearch-rest-client</artifactId>
  <version>${elasticsearch.version}</version>
  </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
  <dependency>
  <groupId>org.apache.kafka</groupId>
  <artifactId>kafka_2.12</artifactId>
@@ -598,6 +521,17 @@
  <artifactId>org.eclipse.paho.client.mqttv3</artifactId>
  <version>${paho.version}</version>
  </dependency>
+
+ <dependency>
+ <groupId>de.ruedigermoeller</groupId>
+ <artifactId>fst</artifactId>
+ <version>${fst.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo</artifactId>
+ <version>${kryo.version}</version>
+ </dependency>
  </dependencies>
  </dependencyManagement>
 

diff --git a/rocketmq-streams-channel-db/pom.xml b/rocketmq-streams-channel-db/pom.xml
@@ -15,7 +15,9 @@
  See the License for the specific language governing permissions and
  limitations under the License.
  -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
  <groupId>org.apache.rocketmq</groupId>

diff --git a/...-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/AbstractMultiTableSink.java b/...-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/AbstractMultiTableSink.java
@@ -34,7 +34,7 @@ public abstract class AbstractMultiTableSink extends EnhanceDBSink {
  protected transient AtomicLong messageCount = new AtomicLong(0);
  protected transient MultiTableSplitFunction<IMessage> multiTableSplitFunction;
 
- public AbstractMultiTableSink(){
+ public AbstractMultiTableSink() {
  }
 
  public AbstractMultiTableSink(String url, String userName, String password) {
@@ -44,16 +44,16 @@ public AbstractMultiTableSink(String url, String userName, String password) {
  }
 
  @Override
- protected boolean initConfigurable(){
+ protected boolean initConfigurable() {
  Iterator<EnhanceDBSink> it = tableSinks.values().iterator();
- while(it.hasNext()){
+ while (it.hasNext()) {
  it.next().initConfigurable();
  }
  return true;
  }
 
  @Override
- public boolean batchAdd(IMessage message, ISplit split) {
+ public boolean batchAdd(IMessage message, ISplit<?, ?> split) {
 
  EnhanceDBSink sink = getOrCreateDBSink(split.getQueueId());
  boolean success = sink.batchAdd(message, split);
@@ -68,7 +68,7 @@ public boolean batchAdd(IMessage message, ISplit split) {
 
  @Override
  public boolean batchAdd(IMessage message) {
- ISplit split = getSplitFromMessage(message);
+ ISplit<?, ?> split = getSplitFromMessage(message);
  return batchAdd(message, split);
  }
 
@@ -142,7 +142,7 @@ protected EnhanceDBSink getOrCreateDBSink(String splitId) {
 
  protected abstract String createTableName(String splitId);
 
- protected abstract ISplit getSplitFromMessage(IMessage message);
+ protected abstract ISplit<?, ?> getSplitFromMessage(IMessage message);
 
  protected class SingleDBSinkCache extends MessageCache<IMessage> {
 

diff --git a/...s-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DynamicMultipleDBSink.java b/...s-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DynamicMultipleDBSink.java
@@ -32,7 +32,7 @@ public class DynamicMultipleDBSink extends AbstractMultiTableSink implements IAf
  String logicTableName;
  String fieldName;
 
- public DynamicMultipleDBSink(){
+ public DynamicMultipleDBSink() {
  }
 
  public String getLogicTableName() {
@@ -63,21 +63,21 @@ protected String createTableName(String splitId) {
  }
 
  @Override
- protected ISplit getSplitFromMessage(IMessage message) {
+ protected ISplit<?, ?> getSplitFromMessage(IMessage message) {
  return this.multiTableSplitFunction.createSplit(message);
  }
 
-
  @Override
  public void doProcessAfterRefreshConfigurable(IConfigurableService configurableService) {
 
- if(this.multiTableSplitFunction == null){
+ if (this.multiTableSplitFunction == null) {
 
  this.multiTableSplitFunction = new MultiTableSplitFunction<IMessage>() {
  @Override
- public ISplit createSplit(IMessage message) {
+ public ISplit<?, ?> createSplit(IMessage message) {
  return new DynamicMultipleDBSplit(message.getMessageBody().getString(fieldName), logicTableName);
  }
+
  @Override
  public String createTableFromSplitId(String splitId) {
  return splitId;
@@ -86,6 +86,5 @@ public String createTableFromSplitId(String splitId) {
 
  }
 
-
  }
 }
diff --git a/...eams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SelfMultiTableSink.java b/...eams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SelfMultiTableSink.java
@@ -41,7 +41,7 @@ protected String createTableName(String splitId) {
  }
 
  @Override
- protected ISplit getSplitFromMessage(IMessage message) {
+ protected ISplit<?, ?> getSplitFromMessage(IMessage message) {
  return multiTableSplitFunction.createSplit(message);
  }
 

diff --git a/...ams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitBySerialNumber.java b/...ams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitBySerialNumber.java
@@ -30,7 +30,7 @@ protected String createTableName(String splitId) {
  }
 
  @Override
- protected ISplit getSplitFromMessage(IMessage message) {
+ protected ISplit<?, ?> getSplitFromMessage(IMessage message) {
  return null;
  }
 }
diff --git a/...annel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitByTimeMultiTableSink.java b/...annel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitByTimeMultiTableSink.java
@@ -30,7 +30,7 @@ protected String createTableName(String splitId) {
  }
 
  @Override
- protected ISplit getSplitFromMessage(IMessage message) {
+ protected ISplit<?, ?> getSplitFromMessage(IMessage message) {
  return null;
  }
 }
diff --git a/rocketmq-streams-channel-es/pom.xml b/rocketmq-streams-channel-es/pom.xml
@@ -1,27 +1,14 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
 
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
  <groupId>org.apache.rocketmq</groupId>
  <artifactId>rocketmq-streams</artifactId>
  <version>1.0.2-preview-SNAPSHOT</version>
- </parent>
+  </parent>
  <artifactId>rocketmq-streams-channel-es</artifactId>
  <name>ROCKETMQ STREAMS :: channel-es</name>
  <packaging>jar</packaging>
@@ -39,6 +26,12 @@
  <groupId>org.apache.rocketmq</groupId>
  <artifactId>rocketmq-streams-configurable</artifactId>
  </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ </dependency>
+
  </dependencies>
  <repositories>
  <repository>