[improve][ci] Split Pulsar IO unit test job to multiple jobs (#20384)

(cherry picked from commit fd36fc1)
apache · May 29, 2023 · 593e55f · 593e55f
1 parent b54a40a
commit 593e55f
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 6 deletions.
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
@@ -202,6 +202,10 @@ jobs:
           - name: Pulsar IO
             group: PULSAR_IO
             timeout: 75
+          - name: Pulsar IO - Elastic Search
+            group: PULSAR_IO_ELASTIC
+          - name: Pulsar IO - Kafka Connect Adaptor
+            group: PULSAR_IO_KAFKA_CONNECT
           - name: Pulsar Client
             group: CLIENT
 

diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
@@ -188,6 +188,18 @@ function test_group_pulsar_io() {
     echo "::endgroup::"
 }
 
+function test_group_pulsar_io_elastic() {
+    echo "::group::Running elastic-search tests"
+    mvn_test --install -Ppulsar-io-elastic-tests,-main
+    echo "::endgroup::"
+}
+
+function test_group_pulsar_io_kafka_connect() {
+    echo "::group::Running Pulsar IO Kafka connect adaptor tests"
+    mvn_test --install -Ppulsar-io-kafka-connect-tests,-main
+    echo "::endgroup::"
+}
+
 function list_test_groups() {
   declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]'
 }

diff --git a/pom.xml b/pom.xml
@@ -2432,6 +2432,20 @@ flexible messaging model and an intuitive client API.</description>
       </modules>
     </profile>
 
+    <profile>
+      <id>pulsar-io-elastic-tests</id>
+      <modules>
+        <module>pulsar-io</module>
+      </modules>
+    </profile>
+
+    <profile>
+      <id>pulsar-io-kafka-connect-tests</id>
+      <modules>
+        <module>pulsar-io</module>
+      </modules>
+    </profile>
+
     <profile>
       <id>pulsar-sql-tests</id>
       <modules>

diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml
@@ -85,22 +85,16 @@
         <module>batch-discovery-triggerers</module>
         <module>batch-data-generator</module>
         <module>common</module>
-        <module>docs</module>
         <module>aws</module>
         <module>twitter</module>
         <module>cassandra</module>
         <module>aerospike</module>
         <module>http</module>
-        <module>kafka</module>
         <module>rabbitmq</module>
         <module>kinesis</module>
         <module>hdfs3</module>
         <module>jdbc</module>
         <module>data-generator</module>
-        <module>elastic-search</module>
-        <module>kafka-connect-adaptor</module>
-        <module>kafka-connect-adaptor-nar</module>
-        <module>debezium</module>
         <module>hdfs2</module>
         <module>canal</module>
         <module>file</module>
@@ -117,6 +111,27 @@
       </modules>
     </profile>
 
+    <profile>
+      <id>pulsar-io-elastic-tests</id>
+      <modules>
+        <module>core</module>
+        <module>common</module>
+        <module>elastic-search</module>
+      </modules>
+    </profile>
+
+    <profile>
+      <id>pulsar-io-kafka-connect-tests</id>
+      <modules>
+        <module>core</module>
+        <module>common</module>
+        <module>kafka</module>
+        <module>kafka-connect-adaptor</module>
+        <module>kafka-connect-adaptor-nar</module>
+        <module>debezium</module>
+      </modules>
+    </profile>
+
     <profile>
       <id>core-modules</id>
       <modules>