diff --git a/adam-core/pom.xml b/adam-core/pom.xml
index dbcdf8d8df..d662127373 100644
--- a/adam-core/pom.xml
+++ b/adam-core/pom.xml
@@ -191,10 +191,6 @@
scalatest_2.10
test
-
- com.amazonaws
- aws-java-sdk
-
org.apache.httpcomponents
httpclient
diff --git a/adam-core/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala b/adam-core/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala
index e662418474..3839b8c2e3 100644
--- a/adam-core/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala
+++ b/adam-core/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala
@@ -24,7 +24,6 @@ trait ADAMFunSuite extends SparkFunSuite {
override val appName: String = "adam"
override val properties: Map[String, String] = Map(("spark.serializer", "org.apache.spark.serializer.KryoSerializer"),
("spark.kryo.registrator", "org.bdgenomics.adam.serialization.ADAMKryoRegistrator"),
- ("spark.kryoserializer.buffer.mb", "4"),
("spark.kryo.referenceTracking", "true"))
}
diff --git a/pom.xml b/pom.xml
index c79f104294..980d038e0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -351,6 +351,12 @@
${utils.version}
test-jar
test
+
+
+ org.apache.spark
+ *
+
+
org.bdgenomics.utils
@@ -364,11 +370,7 @@
com.fasterxml.jackson.core
- jackson-core
-
-
- com.fasterxml.jackson.core
- jackson-annotations
+ *
@@ -478,12 +480,6 @@
slf4j-log4j12
1.7.5
-
- com.amazonaws
- aws-java-sdk
- 1.7.5
- provided
-
org.apache.httpcomponents
httpclient
diff --git a/scripts/jenkins-test b/scripts/jenkins-test
index a44a693f43..409c491946 100755
--- a/scripts/jenkins-test
+++ b/scripts/jenkins-test
@@ -6,11 +6,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$DIR/.."
VERSION="$(grep "" "$PROJECT_ROOT/pom.xml" | head -2 | tail -1 | sed 's/ *//g' | sed 's/<\/version>//g')"
-HADOOP_VERSION="$(grep "" "$PROJECT_ROOT/pom.xml" | head -2 | tail -1 | sed 's/ *//g' | sed 's/<\/hadoop.version>//g')"
-echo "Testing ADAM version ${VERSION} on Hadoop ${HADOOP_VERSION}"
+echo "Testing ADAM version ${VERSION} on Spark ${SPARK_VERSION} and Hadoop ${HADOOP_VERSION}"
export MAVEN_OPTS="-Xmx1536m -XX:MaxPermSize=1g"
-mvn test -P distribution -Dnetworkconnected -Dhadoop.version="${HADOOP_VERSION}"
+mvn test -P distribution -Dnetworkconnected -Dhadoop.version="${HADOOP_VERSION}" -Dspark.version="${SPARK_VERSION}"
ADAM_TMP_DIR="$(mktemp -d -t "adamTestXXXXXXX")"
# Just to be paranoid.. use a directory internal to the ADAM_TMP_DIR
@@ -26,16 +25,24 @@ export SPARK_DRIVER_MEMORY=8g
pushd "$ADAM_TMP_DIR"
-if [[ $HADOOP_VERSION == "1.0.4" ]]; then
- wget -q http://d3kbcqa49mib13.cloudfront.net/spark-1.1.0-bin-hadoop1.tgz
- tar xzvf spark-1.1.0-bin-hadoop1.tgz
- export SPARK_HOME="${ADAM_TMP_DIR}/spark-1.1.0-bin-hadoop1"
+
+if [[ $HADOOP_VERSION =~ ^1\.0 ]]; then
+ HADOOP=hadoop1
+elif [[ $HADOOP_VERSION =~ ^2\.6 ]]; then
+ HADOOP=hadoop2.6
+elif [[ $HADOOP_VERSION =~ ^2\.3 ]]; then
+ HADOOP=hadoop2.3
else
- wget -q http://d3kbcqa49mib13.cloudfront.net/spark-1.1.0-bin-hadoop2.3.tgz
- tar xzvf spark-1.1.0-bin-hadoop2.3.tgz
- export SPARK_HOME="${ADAM_TMP_DIR}/spark-1.1.0-bin-hadoop2.3"
+ echo "Unknown Hadoop version."
+ exit 1
fi
+SPARK="spark-${SPARK_VERSION}"
+
+wget -q http://d3kbcqa49mib13.cloudfront.net/${SPARK}-bin-${HADOOP}.tgz
+tar xzvf ${SPARK}-bin-${HADOOP}.tgz
+export SPARK_HOME="${ADAM_TMP_DIR}/${SPARK}-bin-${HADOOP}"
+
ADAM="./bin/adam-submit"
echo "Fetching BAM file"