spark-defaults.conf

spark.yarn.dist.archives=py3env.zip#temp_envs
spark.yarn.appMasterEnv.PYSPARK_PYTHON=./temp_envs/py3env/bin/python
spark.local.dir=/tmp
spark.master=yarn
spark.shuffle.partitions=300
spark.executor.memory=16g
spark.executor.memoryOverhead=4g
spark.executor.cores=5
spark.driver.cores=5
spark.driver.memory=16g
spark.shuffle.service.enabled=true
spark.sql.broadcastTimeout=3600
spark.dynamicAllocation.enabled=true
spark.dynamicAllocation.minExecutors=1
spark.dynamicAllocation.maxExecutors=30
spark.dynamicAllocation.initialExecutors=10
spark.dynamicAllocation.executorIdleTimeout=240
spark.dynamicAllocation.cachedExecutorIdleTimeout=480