From cb137ad5c21982899bfbe5572f56aa1edfeba6b7 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 14 Apr 2016 01:55:46 -0700 Subject: [PATCH 1/8] [SPARK-14867][BUILD] Make `build/mvn` to use the downloaded maven if it exist. --- build/mvn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/mvn b/build/mvn index eb42552fc499..bd3a200d502b 100755 --- a/build/mvn +++ b/build/mvn @@ -69,7 +69,6 @@ install_app() { # Install maven under the build/ folder install_mvn() { - local MVN_VERSION="3.3.9" local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='} install_app \ @@ -124,10 +123,13 @@ if [ "$1" == "--force" ]; then fi # Install Maven if necessary +MVN_VERSION="3.3.9" MVN_BIN="$(command -v mvn)" if [ ! "$MVN_BIN" -o -n "$FORCE_MVN" ]; then install_mvn +elif [ -f "${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn" ]; then # Try to use downloaded mvn first. + MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn" fi # Install the proper version of Scala and Zinc for the build From 27605baf288181282b7057bbaf3884a9e1062d8a Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 23 Apr 2016 11:39:18 -0700 Subject: [PATCH 2/8] Choose Maven based on versions. --- build/mvn | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/build/mvn b/build/mvn index bd3a200d502b..ba31104942c3 100755 --- a/build/mvn +++ b/build/mvn @@ -69,14 +69,23 @@ install_app() { # Install maven under the build/ folder install_mvn() { - local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='} + MVN_VERSION="3.3.9" + MVN_BIN="$(command -v mvn)" + if [ "$MVN_BIN" ]; then + MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')" + fi + # See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers + function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; } + if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then + local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='} - install_app \ - "${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \ - "apache-maven-${MVN_VERSION}-bin.tar.gz" \ - "apache-maven-${MVN_VERSION}/bin/mvn" + install_app \ + "${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \ + "apache-maven-${MVN_VERSION}-bin.tar.gz" \ + "apache-maven-${MVN_VERSION}/bin/mvn" - MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn" + MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn" + fi } # Install zinc under the build/ folder @@ -115,26 +124,10 @@ install_scala() { # the environment ZINC_PORT=${ZINC_PORT:-"3030"} -# Check for the `--force` flag dictating that `mvn` should be downloaded -# regardless of whether the system already has a `mvn` install -if [ "$1" == "--force" ]; then - FORCE_MVN=1 - shift -fi - -# Install Maven if necessary -MVN_VERSION="3.3.9" -MVN_BIN="$(command -v mvn)" - -if [ ! "$MVN_BIN" -o -n "$FORCE_MVN" ]; then - install_mvn -elif [ -f "${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn" ]; then # Try to use downloaded mvn first. - MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn" -fi - -# Install the proper version of Scala and Zinc for the build +# Install the proper version of Scala, Zinc and Maven for the build install_zinc install_scala +install_mvn # Reset the current working directory cd "${_CALLING_DIR}" From b1cfc12d8fbff4fa109a8e7799d4bb8736ab0ec9 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 23 Apr 2016 11:59:03 -0700 Subject: [PATCH 3/8] Change to local variables. --- build/mvn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/mvn b/build/mvn index ba31104942c3..ad4d1e9fa0e5 100755 --- a/build/mvn +++ b/build/mvn @@ -69,10 +69,10 @@ install_app() { # Install maven under the build/ folder install_mvn() { - MVN_VERSION="3.3.9" + local MVN_VERSION="3.3.9" MVN_BIN="$(command -v mvn)" if [ "$MVN_BIN" ]; then - MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')" + local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')" fi # See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; } From 6fdf92db7c601f5bbed1666e03fce4777a119eac Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 23 Apr 2016 12:03:46 -0700 Subject: [PATCH 4/8] Add a code for backward compatability. --- build/mvn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build/mvn b/build/mvn index ad4d1e9fa0e5..8220752ad0c9 100755 --- a/build/mvn +++ b/build/mvn @@ -124,6 +124,11 @@ install_scala() { # the environment ZINC_PORT=${ZINC_PORT:-"3030"} +# Remove `--force` for backward compatability. +if [ "$1" == "--force" ]; then + shift +fi + # Install the proper version of Scala, Zinc and Maven for the build install_zinc install_scala From 1f4b786d0619b8169ace2a0d101ce29133485b1e Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 23 Apr 2016 20:40:10 -0700 Subject: [PATCH 5/8] Determine the maven version from `pom.xml`. --- build/mvn | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build/mvn b/build/mvn index 8220752ad0c9..b40fe00190bd 100755 --- a/build/mvn +++ b/build/mvn @@ -67,9 +67,11 @@ install_app() { fi } -# Install maven under the build/ folder +# Determine the Maven version from the root pom.xml file and +# install maven under the build/ folder if needed. install_mvn() { - local MVN_VERSION="3.3.9" + local MVN_VERSION=`grep "" "${_DIR}/../pom.xml" | \ + head -1 | cut -f2 -d'>' | cut -f1 -d'<'` MVN_BIN="$(command -v mvn)" if [ "$MVN_BIN" ]; then local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')" @@ -124,7 +126,7 @@ install_scala() { # the environment ZINC_PORT=${ZINC_PORT:-"3030"} -# Remove `--force` for backward compatability. +# Remove `--force` for backward compatibility. if [ "$1" == "--force" ]; then shift fi From d1356b83e211366f44310c3313711e4b46bf2e61 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 24 Apr 2016 12:30:03 -0700 Subject: [PATCH 6/8] Add a deprecation message for option --force --- build/mvn | 1 + 1 file changed, 1 insertion(+) diff --git a/build/mvn b/build/mvn index b40fe00190bd..09f13448165c 100755 --- a/build/mvn +++ b/build/mvn @@ -128,6 +128,7 @@ ZINC_PORT=${ZINC_PORT:-"3030"} # Remove `--force` for backward compatibility. if [ "$1" == "--force" ]; then + echo "WARNING: '--force' is deprecated and ignored." shift fi From 9c4add2f37fc821a9c593e70b09386f7199221f8 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 24 Apr 2016 15:51:23 -0700 Subject: [PATCH 7/8] Update test-dependecies.sh. --- dev/test-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 924b55287c2d..5ea643ee48d9 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -30,7 +30,7 @@ export LC_ALL=C # NOTE: These should match those in the release publishing script HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pyarn -Phive" -MVN="build/mvn --force" +MVN="build/mvn" HADOOP_PROFILES=( hadoop-2.2 hadoop-2.3 From 20a520a35708d6ad6077c78e5e021e4eddbb60a3 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 25 Apr 2016 01:49:06 -0700 Subject: [PATCH 8/8] Use awk command according to the comments. --- build/mvn | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build/mvn b/build/mvn index 09f13448165c..a78b93a68568 100755 --- a/build/mvn +++ b/build/mvn @@ -70,8 +70,7 @@ install_app() { # Determine the Maven version from the root pom.xml file and # install maven under the build/ folder if needed. install_mvn() { - local MVN_VERSION=`grep "" "${_DIR}/../pom.xml" | \ - head -1 | cut -f2 -d'>' | cut -f1 -d'<'` + local MVN_VERSION=`grep "" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'` MVN_BIN="$(command -v mvn)" if [ "$MVN_BIN" ]; then local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')" @@ -108,8 +107,7 @@ install_zinc() { # the build/ folder install_scala() { # determine the Scala version used in Spark - local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | \ - head -1 | cut -f2 -d'>' | cut -f1 -d'<'` + local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'` local scala_bin="${_DIR}/scala-${scala_version}/bin/scala" local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.typesafe.com}