spark-defaults.conf

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Default system properties included when running spark-submit.
# This is useful for setting default environmental settings.

###############################################################################
# Application Properties

# Number of cores to use for the driver process, only in cluster mode.
spark.driver.cores 1

# Limit of total size of serialized results of all partitions for each Spark
# action (e.g. collect). Should be at least 1M, or 0 for unlimited. Jobs will 
# be aborted if the total size is above this limit. Having a high limit may 
# cause out-of-memory errors in driver (depends on spark.driver.memory and 
# memory overhead of objects in JVM). Setting a proper limit can protect the 
# driver from out-of-memory errors.
spark.driver.maxResultSize 1g

# Amount of memory to use for the driver process, i.e. where SparkContext is
# initialized. (e.g. 1g, 2g). 
spark.driver.memory 5g

# Amount of memory to use per executor process (e.g. 2g, 8g).
spark.executor.memory 5g

# Directory to use for "scratch" space in Spark, including map output files
# and RDDs that get stored on disk. This should be on a fast, local disk in
# your system. It can also be a comma-separated list of multiple directories 
# on different disks. 
spark.local.dir /tmp

# Logs the effective SparkConf as INFO when a SparkContext is started
spark.logConf true

# The cluster manager to connect to. See the list of allowed master URL's:
# https://spark.apache.org/docs/latest/submitting-applications.html#master-urls
spark.master local

# If true, restarts the driver automatically if it fails with a non-zero exit
# status. Only has effect in Spark standalone mode or Mesos cluster deploy mode.
spark.driver.supervise false


###############################################################################
# Runtime Environment

# Extra classpath entries to prepend to the classpath of the driver.
spark.driver.extraClassPath /jars/*

# Extra classpath entries to prepend to the classpath of executors. 
spark.executor.extraClassPath /jars/*

# Amount of memory to use per python worker process during aggregation, in the 
# same format as JVM memory strings (e.g. 512m, 2g). If the memory used during 
# aggregation goes above this amount, it will spill the data into disks.
spark.python.worker.memory 512m


###############################################################################
# Shuffle Behavior


###############################################################################
# Spark UI

# Whether to run the web UI for the Spark application.
spark.ui.enabled true

# Whether to log Spark events, useful for reconstructing the Web UI after the
# application has finished.
spark.eventLog.enabled false

# Base directory in which Spark events are logged, if spark.eventLog.enabled is
# true. Within this base directory, Spark creates a sub-directory for each
# application, and logs the events specific to the application in this 
# directory. Users may want to set this to a unified location like an HDFS
# directory so history files can be read by the history server.
spark.eventLog.dir file:///tmp/spark-events