From 6d0573630f9e57814bc48da366ec5827bc1b8456 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:31:14 +0100 Subject: [PATCH] HDDS-11591. Copy dependencies when building each module (#7325) (cherry picked from commit e85b32da2a5fabe7c5f8dafb56f9e8fb1fe6b028) --- hadoop-hdds/pom.xml | 31 ---------------- hadoop-hdds/rocks-native/pom.xml | 2 +- hadoop-hdds/server-scm/pom.xml | 1 + hadoop-ozone/csi/pom.xml | 1 + hadoop-ozone/datanode/pom.xml | 1 + hadoop-ozone/dev-support/checks/dependency.sh | 4 +- .../dev-support/bin/dist-layout-stitching | 10 +++++ hadoop-ozone/dist/pom.xml | 37 +------------------ hadoop-ozone/httpfsgateway/pom.xml | 1 + hadoop-ozone/insight/pom.xml | 2 + hadoop-ozone/ozone-manager/pom.xml | 1 + hadoop-ozone/pom.xml | 31 ---------------- hadoop-ozone/recon/pom.xml | 1 + hadoop-ozone/s3gateway/pom.xml | 1 + hadoop-ozone/tools/pom.xml | 4 ++ pom.xml | 34 +++++++++++++++++ 16 files changed, 61 insertions(+), 101 deletions(-) diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml index 2fb94d8c809a..0237210d2fca 100644 --- a/hadoop-hdds/pom.xml +++ b/hadoop-hdds/pom.xml @@ -322,36 +322,5 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> - - add-classpath-descriptor - - - src/main/java - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - add-classpath-descriptor - prepare-package - - build-classpath - - - ${project.build.outputDirectory}/${project.artifactId}.classpath - $HDDS_LIB_JARS_DIR - true - runtime - - - - - - - diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index 35a03db730a4..4c751e0b10a5 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -385,7 +385,7 @@ maven-dependency-plugin - copy-jars + copy-dependencies process-sources copy-dependencies diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index 26e35b072ffc..4c2e40c37592 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -29,6 +29,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> jar + false diff --git a/hadoop-ozone/csi/pom.xml b/hadoop-ozone/csi/pom.xml index cd4d4eef0419..ba66c5d5272e 100644 --- a/hadoop-ozone/csi/pom.xml +++ b/hadoop-ozone/csi/pom.xml @@ -29,6 +29,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> jar + false true diff --git a/hadoop-ozone/datanode/pom.xml b/hadoop-ozone/datanode/pom.xml index 87664b26614c..733f0837fda9 100644 --- a/hadoop-ozone/datanode/pom.xml +++ b/hadoop-ozone/datanode/pom.xml @@ -27,6 +27,7 @@ 2.0.0-SNAPSHOT + false true true diff --git a/hadoop-ozone/dev-support/checks/dependency.sh b/hadoop-ozone/dev-support/checks/dependency.sh index 116664df81b1..dc95a25e47c2 100755 --- a/hadoop-ozone/dev-support/checks/dependency.sh +++ b/hadoop-ozone/dev-support/checks/dependency.sh @@ -32,8 +32,8 @@ cp ${src_dir}/current.txt "$REPORT_DIR"/ #implementation of sort cli is not exactly the same everywhere. It's better to sort with the same command locally (diff -uw \ - <(sort ${src_dir}/jar-report.txt) \ - <(sort ${src_dir}/current.txt) \ + <(sort -u ${src_dir}/jar-report.txt) \ + <(sort -u ${src_dir}/current.txt) \ || true) \ > "$REPORT_FILE" diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index 0ec066aca563..03905fd20abd 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -71,6 +71,8 @@ run cp -p "${ROOT}/HISTORY.md" . run cp -p "${ROOT}/SECURITY.md" . run cp -p "${ROOT}/CONTRIBUTING.md" . +run mkdir -p ./share/ozone/classpath +run mkdir -p ./share/ozone/lib run mkdir -p ./share/ozone/web run mkdir -p ./bin run mkdir -p ./sbin @@ -129,6 +131,14 @@ run cp -p -r "${ROOT}/hadoop-ozone/dist/target/Dockerfile" . #Copy pre-generated keytabs run cp -p -R "${ROOT}/hadoop-ozone/dist/src/main/keytabs" compose/_keytabs +for file in $(find "${ROOT}" -path '*/target/classes/*.classpath' | sort); do + cp -n -p -v "$file" share/ozone/classpath/ +done + +for file in $(find "${ROOT}" -path '*/share/ozone/lib/*jar' | sort); do + cp -n -p -v "$file" share/ozone/lib/ +done + #workaround for https://issues.apache.org/jira/browse/MRESOURCES-236 find ./compose -name "*.sh" -exec chmod 755 {} \; find ./kubernetes -name "*.sh" -exec chmod 755 {} \; diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index 1a8ab880065b..80efa7a82bec 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -71,24 +71,7 @@ maven-dependency-plugin - copy-classpath-files - prepare-package - - unpack-dependencies - - - - target/ozone-${ozone.version}/share/ozone/classpath - - *.classpath - - hdds-server-scm,ozone-common,ozone-csi,ozone-datanode,ozone-httpfsgateway, - ozone-insight,ozone-manager,ozone-recon,ozone-s3gateway,ozone-tools,hdds-rocks-native,ozone-s3-secret-store - - - - - copy-jars + copy-omitted-jars prepare-package copy-dependencies @@ -98,24 +81,6 @@ runtime - - copy-omitted-jars - prepare-package - - copy - - - target/ozone-${ozone.version}/share/ozone/lib - - - - com.google.protobuf - protobuf-java - ${grpc.protobuf-compile.version} - - - - diff --git a/hadoop-ozone/httpfsgateway/pom.xml b/hadoop-ozone/httpfsgateway/pom.xml index 64eba036a5fa..bacc730a00fc 100644 --- a/hadoop-ozone/httpfsgateway/pom.xml +++ b/hadoop-ozone/httpfsgateway/pom.xml @@ -32,6 +32,7 @@ Apache Ozone HttpFS + false REPO NOT AVAIL REVISION NOT AVAIL yyyy-MM-dd'T'HH:mm:ssZ diff --git a/hadoop-ozone/insight/pom.xml b/hadoop-ozone/insight/pom.xml index c448411c5f6b..fa3862a7f71d 100644 --- a/hadoop-ozone/insight/pom.xml +++ b/hadoop-ozone/insight/pom.xml @@ -27,7 +27,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Ozone Insight Tool Apache Ozone Insight Tool jar + + false diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index ecfd074c40b2..8e78814eb6b4 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -29,6 +29,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> jar + false diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index be928e760549..ed118c89472b 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -455,36 +455,5 @@ - - add-classpath-descriptor - - - src/main/java - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - add-classpath-descriptor - prepare-package - - build-classpath - - - ${project.build.outputDirectory}/${project.artifactId}.classpath - $HDDS_LIB_JARS_DIR - true - runtime - - - - - - - diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index e0ce7065c1c9..026fe01d40bc 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -24,6 +24,7 @@ 4.0.0 ozone-recon + false 8.15.7 diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml index dfcd478b6907..f012d3f1aabb 100644 --- a/hadoop-ozone/s3gateway/pom.xml +++ b/hadoop-ozone/s3gateway/pom.xml @@ -26,6 +26,7 @@ jar 2.0.0-SNAPSHOT + false UTF-8 true diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 96853d8949ca..924408122594 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -28,6 +28,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Ozone Tools jar + + false + + org.apache.ozone diff --git a/pom.xml b/pom.xml index 5c33158666ea..f733f34b7315 100644 --- a/pom.xml +++ b/pom.xml @@ -98,6 +98,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs UTF-8 UTF-8 + true + 3.2.7 bash @@ -1709,6 +1711,38 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + org.apache.maven.plugins + maven-dependency-plugin + + + add-classpath-descriptor + prepare-package + + build-classpath + + + ${project.build.outputDirectory}/${project.artifactId}.classpath + $HDDS_LIB_JARS_DIR + true + runtime + ${classpath.skip} + + + + copy-jars + prepare-package + + copy-dependencies + + + ${project.build.directory}/share/ozone/lib + runtime + ${classpath.skip} + + + + maven-clean-plugin