diff --git a/adam-core/pom.xml b/adam-core/pom.xml index dbcdf8d8df..d662127373 100644 --- a/adam-core/pom.xml +++ b/adam-core/pom.xml @@ -191,10 +191,6 @@ scalatest_2.10 test - - com.amazonaws - aws-java-sdk - org.apache.httpcomponents httpclient diff --git a/adam-core/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala b/adam-core/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala index e662418474..3839b8c2e3 100644 --- a/adam-core/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala +++ b/adam-core/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala @@ -24,7 +24,6 @@ trait ADAMFunSuite extends SparkFunSuite { override val appName: String = "adam" override val properties: Map[String, String] = Map(("spark.serializer", "org.apache.spark.serializer.KryoSerializer"), ("spark.kryo.registrator", "org.bdgenomics.adam.serialization.ADAMKryoRegistrator"), - ("spark.kryoserializer.buffer.mb", "4"), ("spark.kryo.referenceTracking", "true")) } diff --git a/pom.xml b/pom.xml index c79f104294..980d038e0c 100644 --- a/pom.xml +++ b/pom.xml @@ -351,6 +351,12 @@ ${utils.version} test-jar test + + + org.apache.spark + * + + org.bdgenomics.utils @@ -364,11 +370,7 @@ com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations + * @@ -478,12 +480,6 @@ slf4j-log4j12 1.7.5 - - com.amazonaws - aws-java-sdk - 1.7.5 - provided - org.apache.httpcomponents httpclient diff --git a/scripts/jenkins-test b/scripts/jenkins-test index a44a693f43..409c491946 100755 --- a/scripts/jenkins-test +++ b/scripts/jenkins-test @@ -6,11 +6,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" PROJECT_ROOT="$DIR/.." VERSION="$(grep "" "$PROJECT_ROOT/pom.xml" | head -2 | tail -1 | sed 's/ *//g' | sed 's/<\/version>//g')" -HADOOP_VERSION="$(grep "" "$PROJECT_ROOT/pom.xml" | head -2 | tail -1 | sed 's/ *//g' | sed 's/<\/hadoop.version>//g')" -echo "Testing ADAM version ${VERSION} on Hadoop ${HADOOP_VERSION}" +echo "Testing ADAM version ${VERSION} on Spark ${SPARK_VERSION} and Hadoop ${HADOOP_VERSION}" export MAVEN_OPTS="-Xmx1536m -XX:MaxPermSize=1g" -mvn test -P distribution -Dnetworkconnected -Dhadoop.version="${HADOOP_VERSION}" +mvn test -P distribution -Dnetworkconnected -Dhadoop.version="${HADOOP_VERSION}" -Dspark.version="${SPARK_VERSION}" ADAM_TMP_DIR="$(mktemp -d -t "adamTestXXXXXXX")" # Just to be paranoid.. use a directory internal to the ADAM_TMP_DIR @@ -26,16 +25,24 @@ export SPARK_DRIVER_MEMORY=8g pushd "$ADAM_TMP_DIR" -if [[ $HADOOP_VERSION == "1.0.4" ]]; then - wget -q http://d3kbcqa49mib13.cloudfront.net/spark-1.1.0-bin-hadoop1.tgz - tar xzvf spark-1.1.0-bin-hadoop1.tgz - export SPARK_HOME="${ADAM_TMP_DIR}/spark-1.1.0-bin-hadoop1" + +if [[ $HADOOP_VERSION =~ ^1\.0 ]]; then + HADOOP=hadoop1 +elif [[ $HADOOP_VERSION =~ ^2\.6 ]]; then + HADOOP=hadoop2.6 +elif [[ $HADOOP_VERSION =~ ^2\.3 ]]; then + HADOOP=hadoop2.3 else - wget -q http://d3kbcqa49mib13.cloudfront.net/spark-1.1.0-bin-hadoop2.3.tgz - tar xzvf spark-1.1.0-bin-hadoop2.3.tgz - export SPARK_HOME="${ADAM_TMP_DIR}/spark-1.1.0-bin-hadoop2.3" + echo "Unknown Hadoop version." + exit 1 fi +SPARK="spark-${SPARK_VERSION}" + +wget -q http://d3kbcqa49mib13.cloudfront.net/${SPARK}-bin-${HADOOP}.tgz +tar xzvf ${SPARK}-bin-${HADOOP}.tgz +export SPARK_HOME="${ADAM_TMP_DIR}/${SPARK}-bin-${HADOOP}" + ADAM="./bin/adam-submit" echo "Fetching BAM file"