From d09331ebaeb8ae65549d26147ddeb4b6b9d2fee0 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 18 Nov 2024 22:04:13 +0100 Subject: [PATCH 1/3] HDDS-11729. Build fails with -DskipRecon --- CONTRIBUTING.md | 2 +- hadoop-ozone/dist/pom.xml | 19 ++++--------------- hadoop-ozone/pom.xml | 14 ++------------ 3 files changed, 7 insertions(+), 28 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e633b159d22..f8ebce0db06 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ mvn clean verify -DskipTests ### Useful Maven build options * Use `-DskipShade` to skip shaded Ozone FS jar file creation. Saves time, but you can't test integration with other software that uses Ozone as a Hadoop-compatible file system. - * Use `-DskipRecon` to skip building Recon Web UI. It saves about 2 minutes. + * Use `-Dskip.installnodenpm -Dskip.npx` to skip building Recon Web UI. It saves about 2 minutes. * Use `-Pdist` to build the binary tarball, similar to the one that gets released ## Running Ozone in Docker diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index e010ecdb60f..1da959002a3 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -166,6 +166,10 @@ org.apache.ozone hdds-container-service + + org.apache.ozone + ozone-recon + org.apache.ozone ozone-s3gateway @@ -234,21 +238,6 @@ - - build-with-recon - - - !skipRecon - - - - - - org.apache.ozone - ozone-recon - - - diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index ed118c89472..a8b32c686a0 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -38,6 +38,8 @@ ozonefs-common ozonefs datanode + recon + recon-codegen s3gateway dist csi @@ -396,18 +398,6 @@ ozonefs-hadoop2 - - build-with-recon - - - !skipRecon - - - - recon - recon-codegen - - parallel-tests From f7b5b6b1d12bef4fb319412c38e3c1919f9bba29 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 19 Nov 2024 08:20:48 +0100 Subject: [PATCH 2/3] replace skip.installnpx with skip.installnodenpm --- .github/workflows/ci.yml | 2 +- .github/workflows/intermittent-test-check.yml | 2 +- hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +- hadoop-ozone/dev-support/checks/findbugs.sh | 2 +- hadoop-ozone/dev-support/checks/junit.sh | 2 +- hadoop-ozone/dev-support/checks/license.sh | 2 +- hadoop-ozone/dev-support/checks/sonar.sh | 2 +- pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf180b15001..f4de6175c12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -222,7 +222,7 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Compile Ozone using Java ${{ matrix.java }} - run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Dskip.npx -Dskip.installnpx -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }} + run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Dskip.npx -Dskip.installnodenpm -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }} env: OZONE_WITH_COVERAGE: false DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 33a1ee2c6e1..1125759a122 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -115,7 +115,7 @@ jobs: java-version: 8 - name: Build (most) of Ozone run: | - args="-Dskip.npx -Dskip.installnpx -DskipShade -Dmaven.javadoc.skip=true" + args="-Dskip.npx -Dskip.installnodenpm -DskipShade -Dmaven.javadoc.skip=true" if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}" args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}" diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh index 18ae3905975..3d228dde9ac 100755 --- a/hadoop-ozone/dev-support/checks/checkstyle.sh +++ b/hadoop-ozone/dev-support/checks/checkstyle.sh @@ -24,7 +24,7 @@ REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/checkstyle"} mkdir -p "$REPORT_DIR" REPORT_FILE="$REPORT_DIR/summary.txt" -MAVEN_OPTIONS='-B -fae -Dskip.npx -Dskip.installnpx -Dcheckstyle.failOnViolation=false --no-transfer-progress' +MAVEN_OPTIONS='-B -fae -Dskip.npx -Dskip.installnodenpm -Dcheckstyle.failOnViolation=false --no-transfer-progress' declare -i rc mvn ${MAVEN_OPTIONS} checkstyle:check > "${REPORT_DIR}/output.log" diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh index 8c61c524a9e..87404266341 100755 --- a/hadoop-ozone/dev-support/checks/findbugs.sh +++ b/hadoop-ozone/dev-support/checks/findbugs.sh @@ -25,7 +25,7 @@ source "${DIR}/_lib.sh" install_spotbugs -MAVEN_OPTIONS='-B -fae -Dskip.npx -Dskip.installnpx --no-transfer-progress' +MAVEN_OPTIONS='-B -fae -Dskip.npx -Dskip.installnodenpm --no-transfer-progress' if [[ "${OZONE_WITH_COVERAGE}" != "true" ]]; then MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" diff --git a/hadoop-ozone/dev-support/checks/junit.sh b/hadoop-ozone/dev-support/checks/junit.sh index f0002d4eedc..e4f7fb1d8bb 100755 --- a/hadoop-ozone/dev-support/checks/junit.sh +++ b/hadoop-ozone/dev-support/checks/junit.sh @@ -31,7 +31,7 @@ if [[ ${ITERATIONS} -le 0 ]]; then fi export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}" -MAVEN_OPTIONS="-B -V -Dskip.npx -Dskip.installnpx -Dnative.lib.tmp.dir=/tmp --no-transfer-progress" +MAVEN_OPTIONS="-B -V -Dskip.npx -Dskip.installnodenpm -Dnative.lib.tmp.dir=/tmp --no-transfer-progress" if [[ "${OZONE_WITH_COVERAGE}" != "true" ]]; then MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" diff --git a/hadoop-ozone/dev-support/checks/license.sh b/hadoop-ozone/dev-support/checks/license.sh index 90accc85662..0426405fdfe 100755 --- a/hadoop-ozone/dev-support/checks/license.sh +++ b/hadoop-ozone/dev-support/checks/license.sh @@ -42,7 +42,7 @@ DEFAULT_SRC="target/generated-sources/license/THIRD-PARTY.txt" src="${1:-${DEFAULT_SRC}}" if [[ ! -e ${src} ]]; then - MAVEN_OPTIONS="-B -fae -Dskip.npx -Dskip.installnpx --no-transfer-progress ${MAVEN_OPTIONS:-}" + MAVEN_OPTIONS="-B -fae -Dskip.npx -Dskip.installnodenpm --no-transfer-progress ${MAVEN_OPTIONS:-}" mvn ${MAVEN_OPTIONS} license:aggregate-add-third-party | tee "${REPORT_DIR}/output.log" src="${DEFAULT_SRC}" fi diff --git a/hadoop-ozone/dev-support/checks/sonar.sh b/hadoop-ozone/dev-support/checks/sonar.sh index 27a971f691c..8569a0029be 100755 --- a/hadoop-ozone/dev-support/checks/sonar.sh +++ b/hadoop-ozone/dev-support/checks/sonar.sh @@ -24,7 +24,7 @@ if [ ! "$SONAR_TOKEN" ]; then fi -mvn -V -B -DskipShade -DskipTests -Dskip.npx -Dskip.installnpx --no-transfer-progress \ +mvn -V -B -DskipShade -DskipTests -Dskip.npx -Dskip.installnodenpm --no-transfer-progress \ -Dsonar.coverage.jacoco.xmlReportPaths="$(pwd)/target/coverage/all.xml" \ -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=hadoop-ozone \ verify org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.0.1398:sonar diff --git a/pom.xml b/pom.xml index 89a12e7083a..05d0f486448 100644 --- a/pom.xml +++ b/pom.xml @@ -2107,7 +2107,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs void true - true + true true true true From d7b9233a58fcfc33c04cca76166b351e5b4b5ecf Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 19 Nov 2024 18:09:27 +0100 Subject: [PATCH 3/3] keep skipRecon, set properties for skipping frontend-maven-plugin --- .github/workflows/ci.yml | 2 +- .github/workflows/intermittent-test-check.yml | 2 +- CONTRIBUTING.md | 2 +- hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +- hadoop-ozone/dev-support/checks/findbugs.sh | 2 +- hadoop-ozone/dev-support/checks/junit.sh | 2 +- hadoop-ozone/dev-support/checks/license.sh | 2 +- hadoop-ozone/dev-support/checks/sonar.sh | 2 +- pom.xml | 14 ++++++++++++++ 9 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4de6175c12..2947def227d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -222,7 +222,7 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Compile Ozone using Java ${{ matrix.java }} - run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Dskip.npx -Dskip.installnodenpm -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }} + run: hadoop-ozone/dev-support/checks/build.sh -Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }} env: OZONE_WITH_COVERAGE: false DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 1125759a122..cb765f36217 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -115,7 +115,7 @@ jobs: java-version: 8 - name: Build (most) of Ozone run: | - args="-Dskip.npx -Dskip.installnodenpm -DskipShade -Dmaven.javadoc.skip=true" + args="-DskipRecon -DskipShade -Dmaven.javadoc.skip=true" if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}" args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f8ebce0db06..e633b159d22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ mvn clean verify -DskipTests ### Useful Maven build options * Use `-DskipShade` to skip shaded Ozone FS jar file creation. Saves time, but you can't test integration with other software that uses Ozone as a Hadoop-compatible file system. - * Use `-Dskip.installnodenpm -Dskip.npx` to skip building Recon Web UI. It saves about 2 minutes. + * Use `-DskipRecon` to skip building Recon Web UI. It saves about 2 minutes. * Use `-Pdist` to build the binary tarball, similar to the one that gets released ## Running Ozone in Docker diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh index 3d228dde9ac..edab230766c 100755 --- a/hadoop-ozone/dev-support/checks/checkstyle.sh +++ b/hadoop-ozone/dev-support/checks/checkstyle.sh @@ -24,7 +24,7 @@ REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/checkstyle"} mkdir -p "$REPORT_DIR" REPORT_FILE="$REPORT_DIR/summary.txt" -MAVEN_OPTIONS='-B -fae -Dskip.npx -Dskip.installnodenpm -Dcheckstyle.failOnViolation=false --no-transfer-progress' +MAVEN_OPTIONS='-B -fae -DskipRecon -Dcheckstyle.failOnViolation=false --no-transfer-progress' declare -i rc mvn ${MAVEN_OPTIONS} checkstyle:check > "${REPORT_DIR}/output.log" diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh index 87404266341..95555c177f1 100755 --- a/hadoop-ozone/dev-support/checks/findbugs.sh +++ b/hadoop-ozone/dev-support/checks/findbugs.sh @@ -25,7 +25,7 @@ source "${DIR}/_lib.sh" install_spotbugs -MAVEN_OPTIONS='-B -fae -Dskip.npx -Dskip.installnodenpm --no-transfer-progress' +MAVEN_OPTIONS='-B -fae -DskipRecon --no-transfer-progress' if [[ "${OZONE_WITH_COVERAGE}" != "true" ]]; then MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" diff --git a/hadoop-ozone/dev-support/checks/junit.sh b/hadoop-ozone/dev-support/checks/junit.sh index e4f7fb1d8bb..fa3c219c960 100755 --- a/hadoop-ozone/dev-support/checks/junit.sh +++ b/hadoop-ozone/dev-support/checks/junit.sh @@ -31,7 +31,7 @@ if [[ ${ITERATIONS} -le 0 ]]; then fi export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}" -MAVEN_OPTIONS="-B -V -Dskip.npx -Dskip.installnodenpm -Dnative.lib.tmp.dir=/tmp --no-transfer-progress" +MAVEN_OPTIONS="-B -V -DskipRecon -Dnative.lib.tmp.dir=/tmp --no-transfer-progress" if [[ "${OZONE_WITH_COVERAGE}" != "true" ]]; then MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" diff --git a/hadoop-ozone/dev-support/checks/license.sh b/hadoop-ozone/dev-support/checks/license.sh index 0426405fdfe..304b84e539e 100755 --- a/hadoop-ozone/dev-support/checks/license.sh +++ b/hadoop-ozone/dev-support/checks/license.sh @@ -42,7 +42,7 @@ DEFAULT_SRC="target/generated-sources/license/THIRD-PARTY.txt" src="${1:-${DEFAULT_SRC}}" if [[ ! -e ${src} ]]; then - MAVEN_OPTIONS="-B -fae -Dskip.npx -Dskip.installnodenpm --no-transfer-progress ${MAVEN_OPTIONS:-}" + MAVEN_OPTIONS="-B -fae -DskipRecon --no-transfer-progress ${MAVEN_OPTIONS:-}" mvn ${MAVEN_OPTIONS} license:aggregate-add-third-party | tee "${REPORT_DIR}/output.log" src="${DEFAULT_SRC}" fi diff --git a/hadoop-ozone/dev-support/checks/sonar.sh b/hadoop-ozone/dev-support/checks/sonar.sh index 8569a0029be..b9948a31d4e 100755 --- a/hadoop-ozone/dev-support/checks/sonar.sh +++ b/hadoop-ozone/dev-support/checks/sonar.sh @@ -24,7 +24,7 @@ if [ ! "$SONAR_TOKEN" ]; then fi -mvn -V -B -DskipShade -DskipTests -Dskip.npx -Dskip.installnodenpm --no-transfer-progress \ +mvn -V -B -DskipShade -DskipTests -DskipRecon --no-transfer-progress \ -Dsonar.coverage.jacoco.xmlReportPaths="$(pwd)/target/coverage/all.xml" \ -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=hadoop-ozone \ verify org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.0.1398:sonar diff --git a/pom.xml b/pom.xml index 05d0f486448..56ab9252aec 100644 --- a/pom.xml +++ b/pom.xml @@ -2113,6 +2113,20 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs true + + + skip-frontend + + + skipRecon + + + + true + true + + + rocks-native-tests