Skip to content

Commit

Permalink
[ADAM-1317] Add interlocks for move_to_xyz scripts.
Browse files Browse the repository at this point in the history
Resolves #1317.
  • Loading branch information
fnothaft authored and heuermh committed Dec 29, 2016
1 parent 8f6016b commit 0ad63f1
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 13 deletions.
84 changes: 75 additions & 9 deletions scripts/jenkins-test
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ find . -name pom.xml \
-exec sed -i.bak \
-e "s:sun.io.serialization.extendedDebugInfo=true:sun.io.serialization.extendedDebugInfo=true -Djava.io.tmpdir=${ADAM_MVN_TMP_DIR}:g" \
{} \;
find . -name "*.bak" -exec rm {} \;
find . -name "*.bak" -exec rm -f {} \;

# variable declarations
export PATH=${JAVA_HOME}/bin/:${PATH}
Expand All @@ -37,18 +37,59 @@ then
exit 1
fi

# this next line is supposed to fail
set +e

echo "Rewriting POM.xml files to Scala 2.10 and Spark 1 should error..."
./scripts/move_to_spark_1.sh
if [[ $? == 0 ]];
then
echo "Running move_to_spark_1.sh when POMs are set up for Spark 1 should fail, but error code was 0 (success)."
exit 1
fi

./scripts/move_to_scala_2.10.sh
if [[ $? == 0 ]];
then
echo "Running move_to_scala_2.10.sh when POMs are set up for Scala 2.10 should fail, but error code was 0 (success)."
exit 1
fi

set -e

# are we testing for spark 2.0.0? if so, we need to rewrite our poms first
if [ ${SPARK_VERSION} == 2.0.0 ];
then
echo "Rewriting POM.xml files for Scala 2.10."
./scripts/move_to_spark_2.sh

echo "Rewriting POM.xml files for Spark 2."
./scripts/move_to_spark_2.sh

# shouldn't be able to move to spark 2 twice
set +e
./scripts/move_to_spark_2.sh
if [[ $? == 0 ]];
then
echo "We have already moved to Spark 2, so running move_to_spark_2.sh a second time should fail, but error code was 0 (success)."
exit 1
fi
set -e
fi

# are we testing for scala 2.11? if so, we need to rewrite our poms to 2.11 first
if [ ${SCALAVER} == 2.11 ];
then
echo "Rewriting POM.xml files for Scala 2.10."
./scripts/move_to_scala_2.11.sh
echo "Rewriting POM.xml files for Scala 2.11."
./scripts/move_to_scala_2.11.sh

# shouldn't be able to move to scala 2.11 twice
set +e
./scripts/move_to_scala_2.11.sh
if [[ $? == 0 ]];
then
echo "We have already moved to Scala 2.11, so running move_to_scala_2.11.sh a second time should fail, but error code was 0 (success)."
exit 1
fi
set -e
fi

# print versions
Expand Down Expand Up @@ -77,6 +118,33 @@ mvn -U \
find ${ADAM_MVN_TMP_DIR}
rm -rf ${ADAM_MVN_TMP_DIR}

# and move our poms back to their original values
# this will allow us to pass our porcelain test at the end
if [ ${SPARK_VERSION} == 2.0.0 ];
then

echo "Rewriting POM.xml files back to Spark 1."
./scripts/move_to_spark_1.sh
fi
if [ ${SCALAVER} == 2.11 ];
then
echo "Rewriting POM.xml files back to Scala 2.10."
./scripts/move_to_scala_2.10.sh
fi

find . -name pom.xml \
-exec sed -i.bak \
-e "s:sun.io.serialization.extendedDebugInfo=true -Djava.io.tmpdir=${ADAM_MVN_TMP_DIR}:sun.io.serialization.extendedDebugInfo=true:g" \
{} \;
find . -name "*.bak" -exec rm -f {} \;

if test -n "$(git status --porcelain)"
then
echo "Applying move_to_xyz script marred a pom.xml file."
echo "Exiting..."
exit 1
fi

# run integration tests on scala 2.10; prebuilt spark distributions are not available for 2.11
if [ ${SCALAVER} == 2.10 ];
then
Expand Down Expand Up @@ -155,17 +223,15 @@ then
popd

# test that the source is formatted correctly
# we had modified the poms to add a temp dir, so check those out first
# we had modified the poms to add a temp dir, so back out that modification first
pushd ${PROJECT_ROOT}
find . -name pom.xml -exec git checkout {} \;
./scripts/format-source
if test -n "$(git status --porcelain)"
then
echo "Please run './scripts/format-source'"
exit 1
fi
popd

popd
fi

echo
Expand Down
16 changes: 14 additions & 2 deletions scripts/move_to_scala_2.10.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,17 @@

set +x

find . -name "pom.xml" -exec sed -e "s/2.11.8/2.10.6/g" -e "s/2.11/2.10/g" -i.2.10.bak '{}' \;
find . -name "*.2.10.bak" -exec rm {} \;
grep "<scala\.version>" pom.xml | grep -q 2.10
if [[ $? == 0 ]];
then
echo "Scala version is already set to 2.10 (Scala artifacts have _2.10 version suffix in artifact name)."
echo "Cowardly refusing to move to Scala 2.10 a second time..."

exit 1
fi

find . -name "pom.xml" -exec sed -e "s/2.11.8/2.10.6/g" \
-e "s/2.11/2.10/g" -i.2.10.bak \
-e "/no Scala/ s/Scala 2.10/Scala 2.11/g" -i.2.10.bak \
'{}' \;
find . -name "*.2.10.bak" -exec rm -f {} \;
15 changes: 13 additions & 2 deletions scripts/move_to_scala_2.11.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,20 @@

set +x

find . -name "pom.xml" -exec sed -e "s/2.10.6/2.11.8/g" -e "/bdg-utils.version/!s/2.10/2.11/g" -i.2.11.bak '{}' \;
grep "<scala\.version>" pom.xml | grep -q 2.11
if [[ $? == 0 ]];
then
echo "Scala version is already set to 2.11 (Scala artifacts have _2.11 version suffix in artifact name)."
echo "Cowardly refusing to move to Scala 2.11 a second time..."

exit 1
fi

find . -name "pom.xml" -exec sed -e "s/2.10.6/2.11.8/g" \
-e "/bdg-utils.version/! s/2.10/2.11/g" \
-i.2.11.bak '{}' \;
# keep parquet-scala at parquet-scala_2.10
find . -name "pom.xml" -exec sed -e "s/parquet-scala_2.11/parquet-scala_2.10/g" -i.2.11.2.bak '{}' \;
# keep maven-javadoc-plugin at version 2.10.3
find . -name "pom.xml" -exec sed -e "s/2.11.3/2.10.3/g" -i.2.11.3.bak '{}' \;
find . -name "*.2.11.*bak" -exec rm {} \;
find . -name "*.2.11.*bak" -exec rm -f {} \;
9 changes: 9 additions & 0 deletions scripts/move_to_spark_1.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,15 @@

set +x

grep -q "spark2" pom.xml
if [[ $? == 1 ]];
then
echo "POM is already set up for Spark 1 (Spark 1/2 artifacts are missing -spark2 suffix in artifact names)."
echo "Cowardly refusing to move to Spark 1 a second time..."

exit 1
fi

find . -name "pom.xml" -exec sed \
-e "s/-spark2//g" \
-e "/spark.version/ s/2.0.0/1.6.3/g" \
Expand Down
9 changes: 9 additions & 0 deletions scripts/move_to_spark_2.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,15 @@

set +x

grep -q "spark2" pom.xml
if [[ $? == 0 ]];
then
echo "POM is already set up for Spark 2 (Spark 1/2 artifacts have -spark2 suffix in artifact names)."
echo "Cowardly refusing to move to Spark 2 a second time..."

exit 1
fi

svp="\${scala.version.prefix}"
substitution_cmd="s/_$svp/-spark2_$svp/g"

Expand Down

0 comments on commit 0ad63f1

Please sign in to comment.