fix: remove shuffle partitions config

opentargets · Sep 13, 2024 · aff7a69 · aff7a69
1 parent 9a2bc15
commit aff7a69
Show file tree

Hide file tree

Showing 2 changed files with 0 additions and 6 deletions.
diff --git a/src/ot_orchestration/dags/config/finngen_ukb_meta_harmonisation.yaml b/src/ot_orchestration/dags/config/finngen_ukb_meta_harmonisation.yaml
@@ -8,6 +8,3 @@ steps:
       tmp_variant_annotation_path: gs://gentropy-tmp/variant_annotation
       study_index_output_path: gs://finngen_ukb_meta_data/study_index
       summary_stats_output_path: gs://finngen_ukb_meta_data/summary_stats
-session:
-  extended_stark_conf:
-    spark.sql.shuffle.partitions: "3200"
diff --git a/src/ot_orchestration/dags/config/ukb_ppp_eur_harmonisation.yaml b/src/ot_orchestration/dags/config/ukb_ppp_eur_harmonisation.yaml
@@ -8,6 +8,3 @@ steps:
       tmp_variant_annotation_path: gs://gentropy-tmp/variant_annotation
       study_index_output_path: gs://ukb_ppp_eur_data/study_index
       summary_stats_output_path: gs://ukb_ppp_eur_data/summary_stats
-session:
-  extended_stark_conf:
-    spark.sql.shuffle.partitions: "3200"