HDDS-12143. Generate list of integration check splits dynamically

adoroszlai · Jan 28, 2025 · 5488967 · 5488967
1 parent f1b59f1
commit 5488967
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 25 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -43,6 +43,7 @@ jobs:
       GITHUB_CONTEXT: ${{ toJson(github) }}
     outputs:
       acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
+      integration-suites: ${{ steps.integration-suites.outputs.suites }}
       needs-basic-check: ${{ steps.categorize-basic-checks.outputs.needs-basic-check }}
       needs-native-check: ${{ steps.categorize-basic-checks.outputs.needs-native-check }}
       basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
@@ -97,6 +98,9 @@ jobs:
       - name: Acceptance suites
         id: acceptance-suites
         run: dev-support/ci/acceptance_suites.sh
+      - name: Integration suites
+        id: integration-suites
+        run: dev-support/ci/integration_suites.sh
       - name: Categorize Basic Checks
         id: categorize-basic-checks
         env:
@@ -591,17 +595,7 @@ jobs:
     if: needs.build-info.outputs.needs-integration-tests == 'true'
     strategy:
       matrix:
-        profile:
-          - client
-          - container
-          - filesystem
-          - hdds
-          - om
-          - ozone
-          - recon
-          - shell
-          - snapshot
-          - flaky
+        profile: ${{ fromJson(needs.build-info.outputs.integration-suites) }}
       fail-fast: false
     steps:
       - name: Checkout project
@@ -639,7 +633,7 @@ jobs:
             args="$args -DskipShade"
           fi
 
-          hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} ${args}
+          hadoop-ozone/dev-support/checks/integration.sh -Ptest-${{ matrix.profile }} ${args}
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       - name: Summary of failures

diff --git a/dev-support/ci/integration_suites.sh b/dev-support/ci/integration_suites.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# shellcheck source=dev-support/ci/lib/_script_init.sh
+. dev-support/ci/lib/_script_init.sh
+
+# output test suites without test- prefix
+SUITES=$(grep -o '<id>test-[^<]\+' pom.xml \
+  | sort -u | cut -f2 -d'-')
+
+initialization::ga_output suites \
+    "$(initialization::parameters_to_json ${SUITES})"
diff --git a/hadoop-ozone/dev-support/checks/native.sh b/hadoop-ozone/dev-support/checks/native.sh
@@ -19,5 +19,5 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 CHECK=native
 
-source "${DIR}/junit.sh" -Pnative -Drocks_tools_native \
+source "${DIR}/junit.sh" -Pnative-tests -Drocks_tools_native \
   "$@"
diff --git a/pom.xml b/pom.xml
@@ -2165,7 +2165,7 @@
     </profile>
     <!-- The following profiles define test splits for CI -->
     <profile>
-      <id>client</id>
+      <id>test-client</id>
       <build>
         <plugins>
           <plugin>
@@ -2182,7 +2182,7 @@
       </build>
     </profile>
     <profile>
-      <id>container</id>
+      <id>test-container</id>
       <build>
         <plugins>
           <plugin>
@@ -2200,7 +2200,7 @@
       </build>
     </profile>
     <profile>
-      <id>om</id>
+      <id>test-om</id>
       <build>
         <plugins>
           <plugin>
@@ -2220,7 +2220,7 @@
       </build>
     </profile>
     <profile>
-      <id>snapshot</id>
+      <id>test-snapshot</id>
       <build>
         <plugins>
           <plugin>
@@ -2237,7 +2237,7 @@
       </build>
     </profile>
     <profile>
-      <id>filesystem</id>
+      <id>test-filesystem</id>
       <build>
         <plugins>
           <plugin>
@@ -2254,7 +2254,7 @@
       </build>
     </profile>
     <profile>
-      <id>hdds</id>
+      <id>test-hdds</id>
       <build>
         <plugins>
           <plugin>
@@ -2274,7 +2274,7 @@
       </build>
     </profile>
     <profile>
-      <id>ozone</id>
+      <id>test-ozone</id>
       <build>
         <plugins>
           <plugin>
@@ -2301,7 +2301,7 @@
       </build>
     </profile>
     <profile>
-      <id>recon</id>
+      <id>test-recon</id>
       <build>
         <plugins>
           <plugin>
@@ -2318,7 +2318,7 @@
       </build>
     </profile>
     <profile>
-      <id>shell</id>
+      <id>test-shell</id>
       <build>
         <plugins>
           <plugin>
@@ -2337,7 +2337,7 @@
       </build>
     </profile>
     <profile>
-      <id>stable</id>
+      <id>stable-tests</id>
       <build>
         <plugins>
           <plugin>
@@ -2352,7 +2352,7 @@
     </profile>
     <profile>
       <!-- Test split dedicated for running @Flaky tests -->
-      <id>flaky</id>
+      <id>test-flaky</id>
       <build>
         <plugins>
           <plugin>
@@ -2368,7 +2368,7 @@
     </profile>
     <profile>
       <!-- Test split dedicated for running @Native tests -->
-      <id>native</id>
+      <id>native-tests</id>
       <build>
         <plugins>
           <plugin>