Merge pull request #162 from dmwm/fix_cron4eos_dataset

Increase cron4eos_dataset memory
dmwm · Sep 16, 2024 · b8efa0a · b8efa0a
2 parents 08a37ae + 6e00531
commit b8efa0a
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/bin/cron4eos_dataset.sh b/bin/cron4eos_dataset.sh
@@ -42,7 +42,7 @@ util4logi "${myname} Spark Job is starting..."
 spark_submit_args=(
     --master yarn --conf spark.ui.showConsoleProgress=false --conf spark.sql.session.timeZone=UTC
     --conf spark.shuffle.useOldFetchProtocol=true
-    --driver-memory=4g --executor-memory=8g --executor-cores=4 --num-executors=30
+    --driver-memory=16g --executor-memory=8g --executor-cores=4 --num-executors=30
     --conf "spark.driver.bindAddress=0.0.0.0" --conf "spark.driver.host=${K8SHOST}"
     --conf "spark.driver.port=${PORT1}" --conf "spark.driver.blockManager.port=${PORT2}"
     --packages org.apache.spark:spark-avro_2.12:3.4.0