From b572107f8673259b3ace1af78d25bedc96539aef Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Tue, 4 Oct 2022 14:02:56 -0400 Subject: [PATCH 1/9] Check filter set name before running Avro extract [VS-655] --- .../variantstore/wdl/GvsCallsetStatistics.wdl | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/variantstore/wdl/GvsCallsetStatistics.wdl b/scripts/variantstore/wdl/GvsCallsetStatistics.wdl index 71295af64a7..ead68025c48 100644 --- a/scripts/variantstore/wdl/GvsCallsetStatistics.wdl +++ b/scripts/variantstore/wdl/GvsCallsetStatistics.wdl @@ -11,20 +11,21 @@ workflow GvsCallsetStatistics { String statistics_table = "~{extract_prefix}_statistics" } - call CreateTables { + call CheckFilterSetNameAndCreateTables { input: project_id = project_id, dataset_name = dataset_name, metrics_table = metrics_table, aggregate_metrics_table = aggregate_metrics_table, - statistics_table = statistics_table + statistics_table = statistics_table, + filter_set_name = filter_set_name, } # Only collect statistics for the autosomal chromosomes, the first 22 in our location scheme. scatter(chrom in range(22)) { call CollectMetricsForChromosome { input: - go = CreateTables.done, + go = CheckFilterSetNameAndCreateTables.done, project_id = project_id, dataset_name = dataset_name, filter_set_name = filter_set_name, @@ -70,13 +71,14 @@ workflow GvsCallsetStatistics { } } -task CreateTables { +task CheckFilterSetNameAndCreateTables { input { String project_id String dataset_name String metrics_table String aggregate_metrics_table String statistics_table + String filter_set_name } meta { # Always check that these tables exist @@ -85,6 +87,18 @@ task CreateTables { command <<< set -o errexit -o nounset -o xtrace -o pipefail + bq query --location=US --project_id=~{project_id} --format=csv --use_legacy_sql=false ' + SELECT COUNT(*) FROM `~{project_id}.~{dataset_name}.filter_set_info` + WHERE filter_set_name = "~{filter_set_name}" + ' | sed 1d > filter_set_count.txt + + filter_set_count=$(cat filter_set_count.txt) + + if [ $filter_set_count -eq 0 ]; then + echo "Did not find a filter set named '~{filter_set_name}' in '~{project_id}.~{dataset_name}.filter_set_info', exiting." + exit 1 + fi + apk add jq set +o errexit From 3f8328ca37b229106539d02c0cdb0cd84b6e9b06 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Tue, 4 Oct 2022 14:22:42 -0400 Subject: [PATCH 2/9] dockstore --- .dockstore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dockstore.yml b/.dockstore.yml index 0855ac45370..87f4867320a 100644 --- a/.dockstore.yml +++ b/.dockstore.yml @@ -230,7 +230,7 @@ workflows: branches: - master - ah_var_store - - vs_616_split_hail + - vs_655_avro_extract_warn_on_bad_filter_name - name: GvsCallsetStatistics subclass: WDL primaryDescriptorPath: /scripts/variantstore/wdl/GvsCallsetStatistics.wdl From 8d2e10ca83f63b391b8650a539087a20c9ab9841 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Tue, 4 Oct 2022 15:33:53 -0400 Subject: [PATCH 3/9] um --- .dockstore.yml | 2 +- .../variantstore/wdl/GvsCallsetStatistics.wdl | 33 +++++++++---------- .../wdl/GvsExtractAvroFilesForHail.wdl | 11 ++++++- scripts/variantstore/wdl/GvsUtils.wdl | 30 +++++++++++++++++ 4 files changed, 56 insertions(+), 20 deletions(-) diff --git a/.dockstore.yml b/.dockstore.yml index 87f4867320a..2e9d23ae2c4 100644 --- a/.dockstore.yml +++ b/.dockstore.yml @@ -238,7 +238,7 @@ workflows: branches: - master - ah_var_store - - vs_616_split_hail + - vs_655_avro_extract_warn_on_bad_filter_name - name: MitochondriaPipeline subclass: WDL primaryDescriptorPath: /scripts/mitochondria_m2_wdl/MitochondriaPipeline.wdl diff --git a/scripts/variantstore/wdl/GvsCallsetStatistics.wdl b/scripts/variantstore/wdl/GvsCallsetStatistics.wdl index ead68025c48..dd833bdf47d 100644 --- a/scripts/variantstore/wdl/GvsCallsetStatistics.wdl +++ b/scripts/variantstore/wdl/GvsCallsetStatistics.wdl @@ -1,5 +1,7 @@ version 1.0 +import "GvsUtils.wdl" as Utils + workflow GvsCallsetStatistics { input { String project_id @@ -11,21 +13,28 @@ workflow GvsCallsetStatistics { String statistics_table = "~{extract_prefix}_statistics" } - call CheckFilterSetNameAndCreateTables { + call Utils.CheckFilterSetName { + input: + project_id = project_id, + dataset_name = dataset_name, + filter_set_name = filter_set_name + } + + call CreateTables { input: + go = CheckFilterSetName.done, project_id = project_id, dataset_name = dataset_name, metrics_table = metrics_table, aggregate_metrics_table = aggregate_metrics_table, - statistics_table = statistics_table, - filter_set_name = filter_set_name, + statistics_table = statistics_table } # Only collect statistics for the autosomal chromosomes, the first 22 in our location scheme. scatter(chrom in range(22)) { call CollectMetricsForChromosome { input: - go = CheckFilterSetNameAndCreateTables.done, + go = CreateTables.done, project_id = project_id, dataset_name = dataset_name, filter_set_name = filter_set_name, @@ -71,14 +80,14 @@ workflow GvsCallsetStatistics { } } -task CheckFilterSetNameAndCreateTables { +task CreateTables { input { + Boolean go = true String project_id String dataset_name String metrics_table String aggregate_metrics_table String statistics_table - String filter_set_name } meta { # Always check that these tables exist @@ -87,18 +96,6 @@ task CheckFilterSetNameAndCreateTables { command <<< set -o errexit -o nounset -o xtrace -o pipefail - bq query --location=US --project_id=~{project_id} --format=csv --use_legacy_sql=false ' - SELECT COUNT(*) FROM `~{project_id}.~{dataset_name}.filter_set_info` - WHERE filter_set_name = "~{filter_set_name}" - ' | sed 1d > filter_set_count.txt - - filter_set_count=$(cat filter_set_count.txt) - - if [ $filter_set_count -eq 0 ]; then - echo "Did not find a filter set named '~{filter_set_name}' in '~{project_id}.~{dataset_name}.filter_set_info', exiting." - exit 1 - fi - apk add jq set +o errexit diff --git a/scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl b/scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl index ec09869cb66..9476f6f6f9e 100644 --- a/scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl +++ b/scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl @@ -11,7 +11,16 @@ workflow GvsExtractAvroFilesForHail { Int scatter_width = 10 } - call OutputPath { input: go = true } + call Utils.CheckFilterSetName { + input: + project_id = project_id, + dataset_name = dataset, + filter_set_name = filter_set_name, + } + + call OutputPath { + input: go = CheckFilterSetName.done + } call ExtractFromNonSuperpartitionedTables { input: diff --git a/scripts/variantstore/wdl/GvsUtils.wdl b/scripts/variantstore/wdl/GvsUtils.wdl index 9513741aba4..fd7e958b77d 100644 --- a/scripts/variantstore/wdl/GvsUtils.wdl +++ b/scripts/variantstore/wdl/GvsUtils.wdl @@ -419,3 +419,33 @@ task CountSuperpartitions { Int num_superpartitions = read_int('num_superpartitions.txt') } } + +task CheckFilterSetName { + meta { + description: "Makes sure a filter set with the specified name actually exists in the specfied project and dataset, in the process validating the project id and dataset name." + } + input { + String project_id + String dataset_name + String filter_set_name + } + command <<< + bq query --location=US --project_id=~{project_id} --format=csv --use_legacy_sql=false ' + SELECT COUNT(*) FROM `~{project_id}.~{dataset_name}.filter_set_info` + WHERE filter_set_name = "~{filter_set_name}" + ' | sed 1d > filter_set_count.txt + + filter_set_count=$(cat filter_set_count.txt) + + if [ $filter_set_count -eq 0 ]; then + echo "Did not find a filter set named '~{filter_set_name}' in '~{project_id}.~{dataset_name}.filter_set_info', exiting." + exit 1 + fi + >>> + runtime { + docker: "us.gcr.io/broad-dsde-methods/variantstore:2022-09-28-slim" + } + output { + Boolean done = true + } +} \ No newline at end of file From 1ede0f93eba897c156b9d10619b809a1cdc90d51 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Tue, 4 Oct 2022 16:01:26 -0400 Subject: [PATCH 4/9] words --- scripts/variantstore/wdl/GvsUtils.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/variantstore/wdl/GvsUtils.wdl b/scripts/variantstore/wdl/GvsUtils.wdl index fd7e958b77d..f5b684bfe74 100644 --- a/scripts/variantstore/wdl/GvsUtils.wdl +++ b/scripts/variantstore/wdl/GvsUtils.wdl @@ -422,7 +422,7 @@ task CountSuperpartitions { task CheckFilterSetName { meta { - description: "Makes sure a filter set with the specified name actually exists in the specfied project and dataset, in the process validating the project id and dataset name." + description: "Makes sure a filter set with the specified name actually exists in the specfied project and dataset, in the process validating the project id and dataset name as well." } input { String project_id @@ -448,4 +448,4 @@ task CheckFilterSetName { output { Boolean done = true } -} \ No newline at end of file +} From 6b288d8f9b7caec35ef80fe3ea75b2f4096fe653 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Wed, 5 Oct 2022 14:36:32 -0400 Subject: [PATCH 5/9] PR feedback --- .../variantstore/wdl/GvsCallsetStatistics.wdl | 8 +-- .../wdl/GvsExtractAvroFilesForHail.wdl | 8 +-- .../variantstore/wdl/GvsExtractCallset.wdl | 62 +++---------------- scripts/variantstore/wdl/GvsUtils.wdl | 47 +++++++++----- 4 files changed, 49 insertions(+), 76 deletions(-) diff --git a/scripts/variantstore/wdl/GvsCallsetStatistics.wdl b/scripts/variantstore/wdl/GvsCallsetStatistics.wdl index dd833bdf47d..72a3928bc53 100644 --- a/scripts/variantstore/wdl/GvsCallsetStatistics.wdl +++ b/scripts/variantstore/wdl/GvsCallsetStatistics.wdl @@ -13,16 +13,16 @@ workflow GvsCallsetStatistics { String statistics_table = "~{extract_prefix}_statistics" } - call Utils.CheckFilterSetName { + call Utils.ValidateFilterSetName { input: - project_id = project_id, - dataset_name = dataset_name, + data_project = project_id, + data_dataset = dataset_name, filter_set_name = filter_set_name } call CreateTables { input: - go = CheckFilterSetName.done, + go = ValidateFilterSetName.done, project_id = project_id, dataset_name = dataset_name, metrics_table = metrics_table, diff --git a/scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl b/scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl index 9476f6f6f9e..713fb1e0a7a 100644 --- a/scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl +++ b/scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl @@ -11,15 +11,15 @@ workflow GvsExtractAvroFilesForHail { Int scatter_width = 10 } - call Utils.CheckFilterSetName { + call Utils.ValidateFilterSetName { input: - project_id = project_id, - dataset_name = dataset, + data_project = project_id, + data_dataset = dataset, filter_set_name = filter_set_name, } call OutputPath { - input: go = CheckFilterSetName.done + input: go = ValidateFilterSetName.done } call ExtractFromNonSuperpartitionedTables { diff --git a/scripts/variantstore/wdl/GvsExtractCallset.wdl b/scripts/variantstore/wdl/GvsExtractCallset.wdl index 016ed02a030..7e993e3cc8c 100644 --- a/scripts/variantstore/wdl/GvsExtractCallset.wdl +++ b/scripts/variantstore/wdl/GvsExtractCallset.wdl @@ -114,7 +114,7 @@ workflow GvsExtractCallset { } if ( !do_not_filter_override ) { - call ValidateFilterSetName { + call Utils.ValidateFilterSetName { input: query_project = query_project, filter_set_name = filter_set_name, @@ -138,6 +138,7 @@ workflow GvsExtractCallset { call ExtractTask { input: + go = select_first([ValidateFilterSetName.done, true]), dataset_id = dataset_name, call_set_identifier = call_set_identifier, gatk_override = gatk_override, @@ -156,7 +157,6 @@ workflow GvsExtractCallset { fq_filter_set_site_table = fq_filter_set_site_table, fq_filter_set_tranches_table = fq_filter_set_tranches_table, filter_set_name = filter_set_name, - filter_set_name_verified = select_first([ValidateFilterSetName.done, "done"]), drop_state = drop_state, output_file = vcf_filename, output_gcs_dir = output_gcs_dir, @@ -207,52 +207,11 @@ workflow GvsExtractCallset { } } -task ValidateFilterSetName { - input { - String filter_set_name - String data_project - String data_dataset - String query_project - String filter_set_info_timestamp - } - meta { - # Not `volatile: true` since there shouldn't be a need to re-run this if there has already been a successful execution. - } - - # add labels for DSP Cloud Cost Control Labeling and Reporting - String bq_labels = "--label service:gvs --label team:variants --label managedby:extract_callset" - - command <<< - set -ex - - echo "project_id = ~{query_project}" > ~/.bigqueryrc - - OUTPUT=$(bq --location=US --project_id=~{query_project} --format=csv query --use_legacy_sql=false ~{bq_labels} "SELECT filter_set_name as available_filter_set_names FROM \`~{data_project}.~{data_dataset}.filter_set_info\` GROUP BY filter_set_name") - FILTERSETS=${OUTPUT#"available_filter_set_names"} - - if [[ $FILTERSETS =~ "~{filter_set_name}" ]]; then - echo "Filter set name '~{filter_set_name}' found." - else - echo "ERROR: '~{filter_set_name}' is not an existing filter_set_name. Available in ~{data_project}.~{data_dataset} are" - echo $FILTERSETS - exit 1 - fi - >>> - output { - String done = read_string(stdout()) - } - - runtime { - docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:398.0.0" - memory: "3 GB" - disks: "local-disk 10 HDD" - preemptible: 3 - cpu: 1 - } -} task ExtractTask { input { + Boolean go + String dataset_id String call_set_identifier @@ -283,7 +242,6 @@ task ExtractTask { String fq_filter_set_site_table String fq_filter_set_tranches_table String? filter_set_name - String filter_set_name_verified # Runtime Options: File? gatk_override @@ -398,9 +356,9 @@ task SumBytes { print(total_mb);" >>> runtime { - docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:398.0.0" + docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:404.0.0-alpine" memory: "3 GB" - disks: "local-disk 10 HDD" + disks: "local-disk 100 HDD" preemptible: 3 cpu: 1 } @@ -437,9 +395,9 @@ task CreateManifest { } runtime { - docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:398.0.0" + docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:404.0.0-alpine" memory: "3 GB" - disks: "local-disk 10 HDD" + disks: "local-disk 100 HDD" preemptible: 3 cpu: 1 } @@ -479,9 +437,9 @@ task GenerateSampleListFile { } runtime { - docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:398.0.0" + docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:404.0.0-alpine" memory: "3 GB" - disks: "local-disk 10 HDD" + disks: "local-disk 100 HDD" preemptible: 3 cpu: 1 } diff --git a/scripts/variantstore/wdl/GvsUtils.wdl b/scripts/variantstore/wdl/GvsUtils.wdl index f5b684bfe74..164704ca1d0 100644 --- a/scripts/variantstore/wdl/GvsUtils.wdl +++ b/scripts/variantstore/wdl/GvsUtils.wdl @@ -420,32 +420,47 @@ task CountSuperpartitions { } } -task CheckFilterSetName { - meta { - description: "Makes sure a filter set with the specified name actually exists in the specfied project and dataset, in the process validating the project id and dataset name as well." - } +task ValidateFilterSetName { input { - String project_id - String dataset_name + Boolean go = true String filter_set_name + String data_project + String data_dataset + String query_project = data_project + String filter_set_info_timestamp = "" } + meta { + # Not `volatile: true` since there shouldn't be a need to re-run this if there has already been a successful execution. + } + + # add labels for DSP Cloud Cost Control Labeling and Reporting + String bq_labels = "--label service:gvs --label team:variants --label managedby:extract_callset" + command <<< - bq query --location=US --project_id=~{project_id} --format=csv --use_legacy_sql=false ' - SELECT COUNT(*) FROM `~{project_id}.~{dataset_name}.filter_set_info` - WHERE filter_set_name = "~{filter_set_name}" - ' | sed 1d > filter_set_count.txt + set -o errexit -o nounset -o xtrace -o pipefail + + echo "project_id = ~{query_project}" > ~/.bigqueryrc - filter_set_count=$(cat filter_set_count.txt) + OUTPUT=$(bq --location=US --project_id=~{query_project} --format=csv query --use_legacy_sql=false ~{bq_labels} "SELECT filter_set_name as available_filter_set_names FROM \`~{data_project}.~{data_dataset}.filter_set_info\` GROUP BY filter_set_name") + FILTERSETS=${OUTPUT#"available_filter_set_names"} - if [ $filter_set_count -eq 0 ]; then - echo "Did not find a filter set named '~{filter_set_name}' in '~{project_id}.~{dataset_name}.filter_set_info', exiting." + if [[ $FILTERSETS =~ "~{filter_set_name}" ]]; then + echo "Filter set name '~{filter_set_name}' found." + else + echo "ERROR: '~{filter_set_name}' is not an existing filter_set_name. Available in ~{data_project}.~{data_dataset} are" + echo $FILTERSETS exit 1 fi >>> - runtime { - docker: "us.gcr.io/broad-dsde-methods/variantstore:2022-09-28-slim" - } output { Boolean done = true } + + runtime { + docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:398.0.0" + memory: "3 GB" + disks: "local-disk 10 HDD" + preemptible: 3 + cpu: 1 + } } From 286c725a0443d74ced2ae91a95643f74a89431bf Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Wed, 5 Oct 2022 14:39:22 -0400 Subject: [PATCH 6/9] oops --- scripts/variantstore/wdl/GvsExtractCallset.wdl | 6 +++--- scripts/variantstore/wdl/GvsUtils.wdl | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/variantstore/wdl/GvsExtractCallset.wdl b/scripts/variantstore/wdl/GvsExtractCallset.wdl index 7e993e3cc8c..b15a08c7095 100644 --- a/scripts/variantstore/wdl/GvsExtractCallset.wdl +++ b/scripts/variantstore/wdl/GvsExtractCallset.wdl @@ -358,7 +358,7 @@ task SumBytes { runtime { docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:404.0.0-alpine" memory: "3 GB" - disks: "local-disk 100 HDD" + disks: "local-disk 500 HDD" preemptible: 3 cpu: 1 } @@ -397,7 +397,7 @@ task CreateManifest { runtime { docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:404.0.0-alpine" memory: "3 GB" - disks: "local-disk 100 HDD" + disks: "local-disk 500 HDD" preemptible: 3 cpu: 1 } @@ -439,7 +439,7 @@ task GenerateSampleListFile { runtime { docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:404.0.0-alpine" memory: "3 GB" - disks: "local-disk 100 HDD" + disks: "local-disk 500 HDD" preemptible: 3 cpu: 1 } diff --git a/scripts/variantstore/wdl/GvsUtils.wdl b/scripts/variantstore/wdl/GvsUtils.wdl index 164704ca1d0..d2724f4ccd1 100644 --- a/scripts/variantstore/wdl/GvsUtils.wdl +++ b/scripts/variantstore/wdl/GvsUtils.wdl @@ -457,9 +457,9 @@ task ValidateFilterSetName { } runtime { - docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:398.0.0" + docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:404.0.0-alpine" memory: "3 GB" - disks: "local-disk 10 HDD" + disks: "local-disk 500 HDD" preemptible: 3 cpu: 1 } From 66fbedde10368e20257db0060cfb7857bc2e80c3 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Wed, 5 Oct 2022 15:49:14 -0400 Subject: [PATCH 7/9] dockstore --- .dockstore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dockstore.yml b/.dockstore.yml index 2e9d23ae2c4..252512ad7d1 100644 --- a/.dockstore.yml +++ b/.dockstore.yml @@ -208,7 +208,7 @@ workflows: branches: - master - ah_var_store - - vs_648_alpine + - vs_655_avro_extract_warn_on_bad_filter_name - name: GvsIngestTieout subclass: WDL primaryDescriptorPath: /scripts/variantstore/wdl/GvsIngestTieout.wdl From 76da38454d20a8ad09348b318288fec02bd7a8ea Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Wed, 5 Oct 2022 16:48:52 -0400 Subject: [PATCH 8/9] Update scripts/variantstore/wdl/GvsUtils.wdl Co-authored-by: Bec Asch --- scripts/variantstore/wdl/GvsUtils.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/variantstore/wdl/GvsUtils.wdl b/scripts/variantstore/wdl/GvsUtils.wdl index d2724f4ccd1..478fe98d0e3 100644 --- a/scripts/variantstore/wdl/GvsUtils.wdl +++ b/scripts/variantstore/wdl/GvsUtils.wdl @@ -434,7 +434,7 @@ task ValidateFilterSetName { } # add labels for DSP Cloud Cost Control Labeling and Reporting - String bq_labels = "--label service:gvs --label team:variants --label managedby:extract_callset" + String bq_labels = "--label service:gvs --label team:variants --label managedby:gvs_utils" command <<< set -o errexit -o nounset -o xtrace -o pipefail From d0b9568edfadfcc5bc742e1d681c6e613ae9a955 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Thu, 6 Oct 2022 13:44:46 -0400 Subject: [PATCH 9/9] remove location --- scripts/variantstore/wdl/GvsUtils.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/variantstore/wdl/GvsUtils.wdl b/scripts/variantstore/wdl/GvsUtils.wdl index 478fe98d0e3..87106c5ffd1 100644 --- a/scripts/variantstore/wdl/GvsUtils.wdl +++ b/scripts/variantstore/wdl/GvsUtils.wdl @@ -441,7 +441,7 @@ task ValidateFilterSetName { echo "project_id = ~{query_project}" > ~/.bigqueryrc - OUTPUT=$(bq --location=US --project_id=~{query_project} --format=csv query --use_legacy_sql=false ~{bq_labels} "SELECT filter_set_name as available_filter_set_names FROM \`~{data_project}.~{data_dataset}.filter_set_info\` GROUP BY filter_set_name") + OUTPUT=$(bq --project_id=~{query_project} --format=csv query --use_legacy_sql=false ~{bq_labels} "SELECT filter_set_name as available_filter_set_names FROM \`~{data_project}.~{data_dataset}.filter_set_info\` GROUP BY filter_set_name") FILTERSETS=${OUTPUT#"available_filter_set_names"} if [[ $FILTERSETS =~ "~{filter_set_name}" ]]; then