From ab93a8fa7ddb4c8d26543e66671370d6c407fd02 Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Thu, 29 Dec 2022 01:48:59 +0800 Subject: [PATCH] [ETL] use RESOURCES_CONFIGS instead of HEAP_OPTS --- docker/start_etl.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/docker/start_etl.sh b/docker/start_etl.sh index 86e4643cbf..1107cac28c 100755 --- a/docker/start_etl.sh +++ b/docker/start_etl.sh @@ -22,23 +22,21 @@ source "$DOCKER_FOLDER"/docker_build_common.sh declare -r SPARK_VERSION=${SPARK_VERSION:-3.3.1} # ===============================[global variables]================================ declare -r LOCAL_PATH=$(cd -- "$(dirname -- "${DOCKER_FOLDER}")" &>/dev/null && pwd) -declare -r HEAP_OPTS="${HEAP_OPTS:-"-Xmx4G -Xms4G"}" # ===============================[properties keys]================================= declare -r SOURCE_KEY="source.path" # ===============================[spark driver/executor resource]================== -declare -r RESOURCES_CONFIGS="${RESOURCES_CONFIGS:-"2G"}" +declare -r RESOURCES_CONFIGS="${RESOURCES_CONFIGS:-"--conf spark.driver.memory=4g --conf spark.executor.memory=4g"}" # ===================================[functions]=================================== function showHelp() { echo "Usage: [ENV] start_etl.sh properties_path" echo "required Arguments: " - echo " master where to submit spark job" - echo " property.file The path of Spark2Kafka.properties." + echo " master where to submit spark job" + echo " property.file The path of Spark2Kafka.properties." echo "ENV: " - echo " ACCOUNT=skiptests set the account to clone from" - echo " VERSION=main set branch of astraea" - echo " HEAP_OPTS=\"-Xmx2G -Xms2G\" set broker JVM memory" - echo " RESOURCES_CONFIGS=\"-Xmx2G -Xms2G\" set spark memory" + echo " ACCOUNT=skiptests set the account to clone from" + echo " VERSION=main set branch of astraea" + echo " RESOURCES_CONFIGS=\"--conf spark.driver.memory=4g\" set spark memory" } function runContainer() { @@ -92,8 +90,8 @@ function runContainer() { ghcr.io/skiptests/astraea/spark:$SPARK_VERSION \ ./bin/spark-submit \ --conf "spark.ui.port=$ui_port" \ + $RESOURCES_CONFIGS \ --packages org.apache.spark:spark-sql-kafka-0-10_2.13:"$SPARK_VERSION" \ - --executor-memory "$RESOURCES_CONFIGS" \ --class org.astraea.etl.Spark2Kafka \ --jars file:///tmp/astraea-etl.jar \ --master $master \