fix: multiple fixes following test runs

opentargets · Sep 12, 2024 · 9a2bc15 · 9a2bc15
1 parent 67a70c8
commit 9a2bc15
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/src/ot_orchestration/dags/config/finngen_ukb_meta_harmonisation.yaml b/src/ot_orchestration/dags/config/finngen_ukb_meta_harmonisation.yaml
@@ -1,12 +1,13 @@
 cluster_name: otg-finngen-ukb-meta
 steps:
-  - id: otg_finngen_ukb_meta_harmonisation
+  - id: finngen_ukb_meta_ingestion
     params:
-      raw_study_index_path: gs://gentropy-tmp/batch/output/finngen_ukb_meta/study_index.tsv
+      raw_study_index_path_from_tsv: gs://gentropy-tmp/batch/output/finngen_ukb_meta/study_index.tsv
       raw_summary_stats_path: gs://gentropy-tmp/batch/output/finngen_ukb_meta/summary_stats.parquet
       variant_annotation_path: gs://genetics_etl_python_playground/output/python_etl/parquet/XX.XX/variant_annotation
       tmp_variant_annotation_path: gs://gentropy-tmp/variant_annotation
       study_index_output_path: gs://finngen_ukb_meta_data/study_index
       summary_stats_output_path: gs://finngen_ukb_meta_data/summary_stats
-      # session parameters
-      session.extended_stark_conf.spark.sql.shuffle.partitions: "3200"
+session:
+  extended_stark_conf:
+    spark.sql.shuffle.partitions: "3200"
diff --git a/src/ot_orchestration/dags/config/ukb_ppp_eur_harmonisation.yaml b/src/ot_orchestration/dags/config/ukb_ppp_eur_harmonisation.yaml
@@ -2,11 +2,12 @@ cluster_name: otg-ukb-ppp-eur
 steps:
   - id: ukb_ppp_eur_sumstat_preprocess
     params:
-      raw_study_index_path: gs://gentropy-tmp/batch/output/ukb_ppp_eur/study_index.tsv
+      raw_study_index_path_from_tsv: gs://gentropy-tmp/batch/output/ukb_ppp_eur/study_index.tsv
       raw_summary_stats_path: gs://gentropy-tmp/batch/output/ukb_ppp_eur/summary_stats.parquet
       variant_annotation_path: gs://genetics_etl_python_playground/output/python_etl/parquet/XX.XX/variant_annotation
       tmp_variant_annotation_path: gs://gentropy-tmp/variant_annotation
       study_index_output_path: gs://ukb_ppp_eur_data/study_index
       summary_stats_output_path: gs://ukb_ppp_eur_data/summary_stats
-      # session parameters
-      session.extended_stark_conf.spark.sql.shuffle.partitions: "3200"
+session:
+  extended_stark_conf:
+    spark.sql.shuffle.partitions: "3200"