VS-430 Remove YNG status from vds

.dockstore.yml

@@ -155,6 +155,7 @@ workflows:
        branches:
          - master
          - ah_var_store
+         - gg_VS-430_RemoveYNGStatus_From_VDS
        tags:
          - /.*/
    - name: GvsExtractCallset
@@ -298,6 +299,7 @@ workflows:
          branches:
              - master
              - ah_var_store
+             - gg_VS-430_RemoveYNGStatus_From_VDS
          tags:
              - /.*/
    - name: GvsQuickstartIntegration

scripts/variantstore/wdl/GvsCreateVDS.wdl

@@ -3,7 +3,6 @@ version 1.0
 # This WDL will create a VDS in Hail running in a Dataproc cluster.
 import "GvsUtils.wdl" as Utils
 
-
 workflow GvsCreateVDS {
     input {
         String avro_path

scripts/variantstore/wdl/GvsUtils.wdl

@@ -72,7 +72,7 @@ task GetToolVersions {
     # GVS generally uses the smallest `alpine` version of the Google Cloud SDK as it suffices for most tasks, but
     # there are a handlful of tasks that require the larger GNU libc-based `slim`.
     String cloud_sdk_slim_docker = "gcr.io/google.com/cloudsdktool/cloud-sdk:435.0.0-slim"
-    String variants_docker = "us-central1-docker.pkg.dev/broad-dsde-methods/gvs/variants:2024-04-22-alpine-32804b134a75"
+    String variants_docker = "us-central1-docker.pkg.dev/broad-dsde-methods/gvs/variants:2024-06-04-alpine-1dd549733af9"
     String variants_nirvana_docker = "us.gcr.io/broad-dsde-methods/variantstore:nirvana_2022_10_19"
     String gatk_docker = "us-central1-docker.pkg.dev/broad-dsde-methods/gvs/gatk:2024_05_23-gatkbase-28d855745d35"
     String real_time_genomics_docker = "docker.io/realtimegenomics/rtg-tools:latest"

scripts/variantstore/wdl/extract/import_gvs.py

@@ -336,7 +336,11 @@ def convert_array_with_id_keys_to_dense_array(arr, ids, drop=[]):
         vd = vd.annotate_rows(filters=hl.coalesce(site[vd.locus].filters, hl.empty_set(hl.tstr)))
 
         # vqsr ref/alt come in normalized individually, so need to renormalize to the dataset ref allele
+        # For Method 1 - *THIS* is the version ('as_vqsr') that ultimately will be kept - does NOT contain 'yng_status'
         vd = vd.annotate_rows(as_vqsr = hl.dict(vqsr.index(vd.locus, all_matches=True)
+                                                .map(lambda record: (record.alt + vd.alleles[0][hl.len(record.ref):], record.drop('ref', 'alt', 'yng_status')))))
+        # Method 2: Add an 'as_vqsr_dup' from which we will do the calculation of GT filtering ahead, that ultimately will be dropped
+        vd = vd.annotate_rows(as_vqsr_dup = hl.dict(vqsr.index(vd.locus, all_matches=True)
                                                 .map(lambda record: (record.alt + vd.alleles[0][hl.len(record.ref):], record.drop('ref', 'alt')))))
 
         if use_classic_vqsr:
@@ -358,31 +362,35 @@ def convert_array_with_id_keys_to_dense_array(arr, ids, drop=[]):
             is_snp = vd.alleles[1:].map(lambda alt: hl.is_snp(vd.alleles[0], alt))
             vd = vd.annotate_rows(
                 allele_NO=vd.alleles[1:].map(
-                    lambda allele: hl.coalesce(vd.as_vqsr.get(allele).yng_status == 'N', False)),
+                    lambda allele: hl.coalesce(vd.as_vqsr_dup.get(allele).yng_status == 'N', False)),
                 allele_YES=vd.alleles[1:].map(
-                    lambda allele: hl.coalesce(vd.as_vqsr.get(allele).yng_status == 'Y', True)),
+                    lambda allele: hl.coalesce(vd.aas_vqsr_dup.get(allele).yng_status == 'Y', True)),
                 allele_is_snp=is_snp,
                 allele_OK=hl._zip_func(is_snp, vd.alleles[1:],
                                        f=lambda is_snp, alt:
-                                       hl.coalesce(vd.as_vqsr.get(alt).vqslod >=
+                                       hl.coalesce(vd.as_vqsr_dup.get(alt).vqslod >=
                                                    hl.if_else(is_snp, vd.snp_vqslod_threshold, vd.indel_vqslod_threshold),
-                                                   True))
+                                                   True)),
+                as_vqsr_dup=vd.alleles[1:].map(
+                    lambda allele: vd.as_vqsr_dup.get(allele).drop('yng_status'))
             )
         else:
             vd = vd.annotate_globals(truth_sensitivity_snp_threshold=truth_sensitivity_snp_threshold,
                                      truth_sensitivity_indel_threshold=truth_sensitivity_indel_threshold)
             is_snp = vd.alleles[1:].map(lambda alt: hl.is_snp(vd.alleles[0], alt))
             vd = vd.annotate_rows(
                 allele_NO=vd.alleles[1:].map(
-                    lambda allele: hl.coalesce(vd.as_vqsr.get(allele).yng_status == 'N', False)),
+                    lambda allele: hl.coalesce(vd.as_vqsr_dup.get(allele).yng_status == 'N', False)),
                 allele_YES=vd.alleles[1:].map(
-                    lambda allele: hl.coalesce(vd.as_vqsr.get(allele).yng_status == 'Y', True)),
+                    lambda allele: hl.coalesce(vd.as_vqsr_dup.get(allele).yng_status == 'Y', True)),
                 allele_is_snp=is_snp,
                 allele_OK=hl._zip_func(is_snp, vd.alleles[1:],
                                        f=lambda is_snp, alt:
-                                       hl.coalesce(vd.as_vqsr.get(alt).calibration_sensitivity <=
+                                       hl.coalesce(vd.as_vqsr_dup.get(alt).calibration_sensitivity <=
                                                    hl.if_else(is_snp, vd.truth_sensitivity_snp_threshold, vd.truth_sensitivity_indel_threshold),
-                                                   True))
+                                                   True)),
+                as_vqsr_dup=vd.alleles[1:].map(
+                    lambda allele: vd.as_vqsr_dup.get(allele).drop('yng_status'))
             )
 
         lgt = vd.LGT