feat(ci): split integration tests execution by category (#2548)

* feat(build): split ITs by category, parallel CRD generation Also avoid generating CRDs when not needed by default, summarizes test results. * chore(deps): bump actions-setup-minikube to 2.13.0 * feat: default to use vertx client * chore(ci): reduce tested combinations, only run client tests on baseapi * fix: increase deletion timeout --------- Signed-off-by: Chris Laprun <claprun@redhat.com>
operator-framework · Oct 17, 2024 · 22a2b21 · 22a2b21
1 parent 3dd77c1
commit 22a2b21
Show file tree

Hide file tree

Showing 9 changed files with 196 additions and 97 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,49 @@
+name: Build
+
+env:
+  MAVEN_ARGS: -V -ntp -e
+
+on:
+  workflow_call:
+
+jobs:
+  integration_tests:
+    strategy:
+      matrix:
+        java: [ 17, 21 ]
+        kubernetes: [ 'v1.28.14', 'v1.29.9','1.30.5', '1.31.1' ]
+        it-category: [ 'baseapi', 'dependent', 'workflow' ]
+    uses: ./.github/workflows/integration-tests.yml
+    with:
+      java-version: ${{ matrix.java }}
+      kube-version: ${{ matrix.kubernetes }}
+      it-category: ${{ matrix.it-category }}
+
+  http_client_tests:
+    strategy:
+      matrix:
+        java: [ 17, 21 ]
+        kubernetes: [ 'v1.28.14', 'v1.29.9','1.30.5', '1.31.1' ]
+        it-category: [ 'baseapi' ]
+        httpclient: [ 'vertx', 'jdk', 'jetty' ]
+    uses: ./.github/workflows/integration-tests.yml
+    with:
+      java-version: ${{ matrix.java }}
+      kube-version: ${{ matrix.kubernetes }}
+      it-category: ${{ matrix.it-category }}
+      http-client: ${{ matrix.httpclient }}
+
+  special_integration_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [ 17, 21 ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Java and Maven
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java }}
+      - name: Run Special Integration Tests
+        run: ./mvnw ${MAVEN_ARGS} -B package -P minimal-watch-timeout-dependent-it --file pom.xml
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -30,7 +30,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Minikube-Kubernetes
-        uses: manusa/actions-setup-minikube@v2.12.0
+        uses: manusa/actions-setup-minikube@v2.13.0
         with:
           minikube version: v1.33.0
           kubernetes version: v1.31.0

diff --git a/.github/workflows/fabric8-next-version-schedule.yml b/.github/workflows/fabric8-next-version-schedule.yml
@@ -23,46 +23,8 @@ jobs:
         with:
           distribution: temurin
           java-version: 17
-          cache: 'maven'
       - name: Run unit tests
         run: ./mvnw ${MAVEN_ARGS} clean install --file pom.xml
 
-  integration_tests:
-    strategy:
-      matrix:
-        java: [ 11, 17 ]
-        kubernetes: ['v1.28.12', 'v1.29.7','1.30.3', '1.31.0']
-    uses: ./.github/workflows/integration-tests.yml
-    with:
-      java-version: ${{ matrix.java }}
-      kube-version: ${{ matrix.kubernetes }}
-
-  httpclient-tests:
-    strategy:
-      matrix:
-        httpclient: [ 'vertx', 'jdk', 'jetty' ]
-    uses: ./.github/workflows/integration-tests.yml
-    with:
-      java-version: 17
-      kube-version: 'v1.29.1'
-      http-client: ${{ matrix.httpclient }}
-      experimental: true
-      checkout-ref: 'fabric8-next-version'
-
-  special_integration_tests:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java: [ 11, 17 ]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: 'fabric8-next-version'
-      - name: Set up Java and Maven
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.java }}
-          cache: 'maven'
-      - name: Run Special Integration Tests
-        run: ./mvnw ${MAVEN_ARGS} -B package -P minimal-watch-timeout-dependent-it --file pom.xml
+  build:
+    uses: ./.github/workflows/build.yml
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -12,7 +12,7 @@ on:
       http-client:
         type: string
         required: false
-        default: 'jdk'
+        default: 'vertx'
       experimental:
         type: boolean
         required: false
@@ -21,6 +21,10 @@ on:
         type: string
         required: false
         default: ''
+      it-category:
+        type: string
+        required: false
+        default: ''
 
 jobs:
   integration_tests:
@@ -40,11 +44,19 @@ jobs:
           java-version: ${{ inputs.java-version }}
           cache: 'maven'
       - name: Set up Minikube
-        uses: manusa/actions-setup-minikube@v2.12.0
+        uses: manusa/actions-setup-minikube@v2.13.0
         with:
-          minikube version: v1.33.0
-          kubernetes version: ${{ inputs.kube-version }}
+          minikube version: 'v1.33.0'
+          kubernetes version: '${{ inputs.kube-version }}'
           driver: 'docker'
           github token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Run integration tests
-        run: ./mvnw ${MAVEN_ARGS} -B package -P no-unit-tests -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml
+      - name: "${{inputs.it-category}} integration tests (kube: ${{ inputs.kube-version }} / java: ${{ inputs.java-version }} / client: ${{ inputs.http-client }})"
+        run: |
+          if [ -z "${{inputs.it-category}}" ]; then
+            it_profile="integration-tests"
+          else
+            it_profile="integration-tests-${{inputs.it-category}}"
+          fi
+          echo "Using profile: ${it_profile}"
+          ./mvnw ${MAVEN_ARGS} -T1C -B install -DskipTests -Pno-apt --file pom.xml
+          ./mvnw ${MAVEN_ARGS} -T1C -B package -P${it_profile} -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -28,41 +28,7 @@ jobs:
         run: |
           ./mvnw ${MAVEN_ARGS} spotless:check --file pom.xml
       - name: Run unit tests
-        run: ./mvnw ${MAVEN_ARGS} clean install --file pom.xml
+        run: ./mvnw ${MAVEN_ARGS} clean install -Pno-apt --file pom.xml
 
-  integration_tests:
-    strategy:
-      matrix:
-        java: [ 17, 21 ]
-        kubernetes: [ 'v1.28.12', 'v1.29.7','1.30.3', '1.31.0' ]
-    uses: ./.github/workflows/integration-tests.yml
-    with:
-      java-version: ${{ matrix.java }}
-      kube-version: ${{ matrix.kubernetes }}
-
-  httpclient-tests:
-    strategy:
-      matrix:
-        httpclient: [ 'vertx', 'jdk', 'jetty' ]
-    uses: ./.github/workflows/integration-tests.yml
-    with:
-      java-version: 17
-      kube-version: 'v1.29.1'
-      http-client: ${{ matrix.httpclient }}
-      experimental: true
-
-  special_integration_tests:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java: [ 17, 21 ]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Java and Maven
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.java }}
-          cache: 'maven'
-      - name: Run Special Integration Tests
-        run: ./mvnw ${MAVEN_ARGS} -B package -P minimal-watch-timeout-dependent-it --file pom.xml
+  build:
+    uses: ./.github/workflows/build.yml
diff --git a/...javaoperatorsdk/operator/processing/dependent/workflow/WorkflowReconcileExecutorTest.java b/...javaoperatorsdk/operator/processing/dependent/workflow/WorkflowReconcileExecutorTest.java
@@ -6,6 +6,9 @@
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import io.fabric8.kubernetes.api.model.ConfigMap;
 import io.fabric8.kubernetes.api.model.HasMetadata;
@@ -20,6 +23,7 @@
 import static org.mockito.Mockito.*;
 
 class WorkflowReconcileExecutorTest extends AbstractWorkflowExecutorTest {
+  private static final Logger log = LoggerFactory.getLogger(WorkflowReconcileExecutorTest.class);
 
   @SuppressWarnings("unchecked")
   Context<TestCustomResource> mockContext = mock(Context.class);
@@ -30,7 +34,8 @@ class WorkflowReconcileExecutorTest extends AbstractWorkflowExecutorTest {
 
   @BeforeEach
   @SuppressWarnings("unchecked")
-  void setup() {
+  void setup(TestInfo testInfo) {
+    log.debug("==> Starting test {}", testInfo.getDisplayName());
     when(mockContext.getWorkflowExecutorService()).thenReturn(executorService);
     when(mockContext.eventSourceRetriever()).thenReturn(mock(EventSourceRetriever.class));
   }

diff --git a/operator-framework/pom.xml b/operator-framework/pom.xml
@@ -57,11 +57,6 @@
       <artifactId>compile-testing</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>io.fabric8</groupId>
-      <artifactId>crd-generator-apt</artifactId>
-      <scope>test</scope>
-    </dependency>
     <!-- This is just here to make some special test using Route, Openshift is actually not used in tests -->
     <dependency>
       <groupId>io.fabric8</groupId>
@@ -113,7 +108,10 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+      </plugin>
     </plugins>
   </build>
-
 </project>
diff --git a/...va/io/javaoperatorsdk/operator/baseapi/clusterscopedresource/ClusterScopedResourceIT.java b/...va/io/javaoperatorsdk/operator/baseapi/clusterscopedresource/ClusterScopedResourceIT.java
@@ -9,6 +9,7 @@
 import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
 import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension;
 
+import static io.javaoperatorsdk.operator.IntegrationTestConstants.GARBAGE_COLLECTION_TIMEOUT_SECONDS;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
 
@@ -46,7 +47,7 @@ void crudOperationOnClusterScopedCustomResource() {
     });
 
     operator.delete(resource);
-    await().atMost(Duration.ofSeconds(30))
+    await().atMost(Duration.ofSeconds(GARBAGE_COLLECTION_TIMEOUT_SECONDS))
         .untilAsserted(() -> assertThat(operator.get(ConfigMap.class, TEST_NAME)).isNull());
   }