diff --git a/bin/adam-pyspark b/bin/adam-pyspark index 6026e74327..c2b9491b1c 100755 --- a/bin/adam-pyspark +++ b/bin/adam-pyspark @@ -36,6 +36,9 @@ fi # submit the job to Spark "$SPARK_SHELL" \ - --properties-file "$ADAM_REPO"/bin/adam-spark-defaults.conf \ + --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \ + --conf spark.kryo.registrator=org.bdgenomics.adam.serialization.ADAMKryoRegistrator \ + --conf spark.kryoserializer.buffer.mb=4 \ + --conf spark.kryo.referenceTracking=true \ --jars "$ADAM_JARS" \ "$@" diff --git a/bin/adam-shell b/bin/adam-shell index 206461eedc..f742776f0a 100755 --- a/bin/adam-shell +++ b/bin/adam-shell @@ -36,6 +36,9 @@ fi # submit the job to Spark "$SPARK_SHELL" \ - --properties-file "$ADAM_REPO"/bin/adam-spark-defaults.conf \ + --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \ + --conf spark.kryo.registrator=org.bdgenomics.adam.serialization.ADAMKryoRegistrator \ + --conf spark.kryoserializer.buffer.mb=4 \ + --conf spark.kryo.referenceTracking=true \ --jars "$ADAM_JARS" \ "$@" diff --git a/bin/adam-spark-defaults.conf b/bin/adam-spark-defaults.conf deleted file mode 100644 index 87e36a5037..0000000000 --- a/bin/adam-spark-defaults.conf +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to Big Data Genomics (BDG) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The BDG licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -spark.serializer org.apache.spark.serializer.KryoSerializer -spark.kryo.registrator org.bdgenomics.adam.serialization.ADAMKryoRegistrator -spark.kryoserializer.buffer.mb 4 -spark.kryo.referenceTracking true diff --git a/bin/adam-submit b/bin/adam-submit index 1341209322..3712c95604 100755 --- a/bin/adam-submit +++ b/bin/adam-submit @@ -49,7 +49,10 @@ fi # submit the job to Spark "$SPARK_SUBMIT" \ --class org.bdgenomics.adam.cli.ADAMMain \ - --properties-file "$ADAM_REPO"/bin/adam-spark-defaults.conf \ + --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \ + --conf spark.kryo.registrator=org.bdgenomics.adam.serialization.ADAMKryoRegistrator \ + --conf spark.kryoserializer.buffer.mb=4 \ + --conf spark.kryo.referenceTracking=true \ --jars "$ADAM_JARS" \ "$ADAM_CLI_JAR" \ "$@" diff --git a/scripts/jenkins-test b/scripts/jenkins-test index 8e1c61615f..8f6992c258 100755 --- a/scripts/jenkins-test +++ b/scripts/jenkins-test @@ -27,13 +27,13 @@ export SPARK_DRIVER_MEMORY=8g pushd $ADAM_TMP_DIR if [[ $HADOOP_VERSION == "1.0.4" ]]; then - wget http://d3kbcqa49mib13.cloudfront.net/spark-1.0.1-bin-hadoop1.tgz - tar xzvf spark-1.0.1-bin-hadoop1.tgz - export SPARK_HOME="${ADAM_TMP_DIR}/spark-1.0.1-bin-hadoop1" + wget http://d3kbcqa49mib13.cloudfront.net/spark-1.1.0-bin-hadoop1.tgz + tar xzvf spark-1.1.0-bin-hadoop1.tgz + export SPARK_HOME="${ADAM_TMP_DIR}/spark-1.1.0-bin-hadoop1" else - wget http://d3kbcqa49mib13.cloudfront.net/spark-1.0.1-bin-hadoop2.tgz - tar xzvf spark-1.0.1-bin-hadoop2.tgz - export SPARK_HOME="${ADAM_TMP_DIR}/spark-1.0.1-bin-hadoop2" + wget http://d3kbcqa49mib13.cloudfront.net/spark-1.1.0-bin-hadoop2.3.tgz + tar xzvf spark-1.1.0-bin-hadoop2.3.tgz + export SPARK_HOME="${ADAM_TMP_DIR}/spark-1.1.0-bin-hadoop2.3" fi ADAM="./bin/adam-submit "