Merge branch 'master' into HDDS-11531

apache · Dec 8, 2024 · 038cea7 · 038cea7
2 parents 57ea5fd + 51c6ed6
commit 038cea7
Show file tree

Hide file tree

Showing 945 changed files with 26,334 additions and 8,276 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -28,9 +28,12 @@ on:
         required: false
 env:
   FAIL_FAST: ${{ github.event_name == 'pull_request' }}
-  BUILD_JAVA_VERSION: 8 # minimum version
-  TEST_JAVA_VERSION: 17 # preferred version
+  # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
+  TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  HADOOP_IMAGE: ghcr.io/apache/hadoop
+  OZONE_IMAGE: ghcr.io/apache/ozone
+  OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
   OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}
 jobs:
   build-info:
@@ -104,10 +107,6 @@ jobs:
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     if: needs.build-info.outputs.needs-build == 'true'
-    strategy:
-      matrix:
-        java: [ 8, 17 ]
-      fail-fast: false
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -138,33 +137,32 @@ jobs:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: ${{ matrix.java }}
+          java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Run a full build
         run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Store binaries for tests
         uses: actions/upload-artifact@v4
         with:
-          name: ozone-bin-${{ matrix.java }}
+          name: ozone-bin
           path: |
             hadoop-ozone/dist/target/ozone-*.tar.gz
             !hadoop-ozone/dist/target/ozone-*-src.tar.gz
           retention-days: 1
       - name: Store source tarball for compilation
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.java == env.BUILD_JAVA_VERSION }}
+        if: needs.build-info.outputs.needs-compile == 'true'
         with:
           name: ozone-src
           path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
           retention-days: 1
       - name: Store Maven repo for tests
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.java == env.BUILD_JAVA_VERSION }}
         with:
           name: ozone-repo
           path: |
@@ -175,6 +173,8 @@ jobs:
       - build-info
       - build
       - basic
+      - dependency
+      - license
     timeout-minutes: 45
     if: needs.build-info.outputs.needs-compile == 'true'
     strategy:
@@ -183,7 +183,7 @@ jobs:
         include:
           - os: ubuntu-20.04
           - java: 8
-            os: macos-12
+            os: macos-13
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:
@@ -216,13 +216,13 @@ jobs:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java ${{ matrix.java }}
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}
       - name: Compile Ozone using Java ${{ matrix.java }}
-        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Dskip.npx -Dskip.installnpx -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
+        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
         env:
           OZONE_WITH_COVERAGE: false
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
@@ -265,19 +265,18 @@ jobs:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java 8
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: 8
       - name: Execute tests
         run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh ${{ inputs.ratis_args }}
-        continue-on-error: true
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ matrix.check }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
@@ -313,19 +312,18 @@ jobs:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: 8
+          java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
         run: hadoop-ozone/dev-support/checks/${{ github.job }}.sh ${{ inputs.ratis_args }}
-        continue-on-error: true
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
@@ -348,7 +346,7 @@ jobs:
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
-          name: ozone-bin-${{ env.BUILD_JAVA_VERSION }}
+          name: ozone-bin
       - name: Untar binaries
         run: |
           mkdir dist
@@ -392,13 +390,76 @@ jobs:
           name: ozone-repo
           path: |
             ~/.m2/repository/org/apache/ozone
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
         run: |
           hadoop-ozone/dev-support/checks/${{ github.job }}.sh
+      - name: Summary of failures
+        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
+        if: ${{ failure() }}
+      - name: Archive build results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: ${{ github.job }}
+          path: target/${{ github.job }}
         continue-on-error: true
+  repro:
+    needs:
+      - build-info
+      - build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
+      - name: Download Ozone repo
+        id: download-ozone-repo
+        uses: actions/download-artifact@v4
+        with:
+          name: ozone-repo
+          path: |
+            ~/.m2/repository/org/apache/ozone
+      - name: Download Ratis repo
+        if: ${{ inputs.ratis_args != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ env.TEST_JAVA_VERSION }}
+      - name: Execute tests
+        run: |
+          hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
+      - name: Install diffoscope
+        run: |
+          sudo apt update -q
+          sudo apt install -y diffoscope
+        if: ${{ failure() }}
+      - name: Check artifact differences
+        run: |
+          hadoop-ozone/dev-support/checks/_diffoscope.sh
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -411,6 +472,8 @@ jobs:
       - build-info
       - build
       - basic
+      - dependency
+      - license
     runs-on: ubuntu-20.04
     timeout-minutes: 150
     if: needs.build-info.outputs.needs-compose-tests == 'true'
@@ -425,14 +488,8 @@ jobs:
           ref: ${{ needs.build-info.outputs.sha }}
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
-        if: ${{ matrix.suite }} == 'MR' # until HDDS-11053 is fixed
-        with:
-          name: ozone-bin-${{ env.BUILD_JAVA_VERSION }}
-      - name: Download compiled Ozone binaries
-        uses: actions/download-artifact@v4
-        if: ${{ matrix.suite }} != 'MR'
         with:
-          name: ozone-bin-${{ env.TEST_JAVA_VERSION }}
+          name: ozone-bin
       - name: Untar binaries
         run: |
           mkdir -p hadoop-ozone/dist/target
@@ -448,10 +505,9 @@ jobs:
           KEEP_IMAGE: false
           OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
           OZONE_VOLUME_OWNER: 1000
-        continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -464,6 +520,8 @@ jobs:
       - build-info
       - build
       - basic
+      - dependency
+      - license
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
@@ -475,7 +533,7 @@ jobs:
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
-          name: ozone-bin-${{ env.BUILD_JAVA_VERSION }}
+          name: ozone-bin
       - name: Untar binaries
         run: |
           mkdir -p hadoop-ozone/dist/target
@@ -486,10 +544,9 @@ jobs:
           sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
           popd
           ./hadoop-ozone/dev-support/checks/kubernetes.sh
-        continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -545,19 +602,25 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
-        continue-on-error: true
         run: |
           args="${{ inputs.ratis_args }}"
           if [[ "${{ matrix.profile }}" == "flaky" ]]; then
             args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600"
           fi
+          if [[ "${{ matrix.profile }}" != "filesystem" ]]; then
+            args="$args -DskipShade"
+          fi
 
           hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} ${args}
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
-        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        run: |
+          if [[ -s "target/${{ github.job }}/summary.md" ]]; then
+            cat target/${{ github.job }}/summary.md >> $GITHUB_STEP_SUMMARY
+          fi
+          hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -596,7 +659,7 @@ jobs:
       - name: Untar binaries
         run: |
           mkdir -p hadoop-ozone/dist/target
-          tar xzvf target/artifacts/ozone-bin-${{ env.TEST_JAVA_VERSION }}/ozone*.tar.gz -C hadoop-ozone/dist/target
+          tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target
       - name: Setup java ${{ env.TEST_JAVA_VERSION }}
         uses: actions/setup-java@v4
         with:

diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml
@@ -115,7 +115,7 @@ jobs:
           java-version: 8
       - name: Build (most) of Ozone
         run: |
-          args="-Dskip.npx -Dskip.installnpx -DskipShade -Dmaven.javadoc.skip=true"
+          args="-DskipRecon -DskipShade -Dmaven.javadoc.skip=true"
           if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
             args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
             args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"
@@ -183,7 +183,7 @@ jobs:
             export OZONE_REPO_CACHED=true
           fi
 
-          args="-DexcludedGroups=native|slow|unhealthy"
+          args="-DexcludedGroups=native|slow|unhealthy -DskipShade"
           if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
             args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
             args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"

diff --git a/.github/workflows/populate-cache.yml b/.github/workflows/populate-cache.yml
@@ -26,8 +26,7 @@ on:
       - 'pom.xml'
       - '**/pom.xml'
       - '.github/workflows/populate-cache.yml'
-  schedule:
-    - cron: '20 3 * * *'
+  workflow_call:
   workflow_dispatch:
 
 jobs:

diff --git a/.github/workflows/scheduled-cache-update.yml b/.github/workflows/scheduled-cache-update.yml
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow periodically updates dependency cache.
+
+name: scheduled-cache-update
+
+on:
+  schedule:
+    - cron: '20 3 * * *'
+
+jobs:
+  update:
+    uses: ./.github/workflows/populate-cache.yml
+    secrets: inherit
diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml
@@ -29,6 +29,6 @@
   <extension>
     <groupId>com.gradle</groupId>
     <artifactId>common-custom-user-data-maven-extension</artifactId>
-    <version>2.0</version>
+    <version>2.0.1</version>
   </extension>
 </extensions>