From ca8771de2159e62a7a9e4f7b436b5e1136f7ef21 Mon Sep 17 00:00:00 2001 From: Thomas van Latum Date: Wed, 21 Jun 2023 21:16:03 +0200 Subject: [PATCH 1/2] fix check schema --- .../plugins/bigquery/create_external_schema.sql | 17 ++++++++++++++--- run_test.sh | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/macros/plugins/bigquery/create_external_schema.sql b/macros/plugins/bigquery/create_external_schema.sql index a6ae1eb2..1199dfb0 100644 --- a/macros/plugins/bigquery/create_external_schema.sql +++ b/macros/plugins/bigquery/create_external_schema.sql @@ -7,9 +7,20 @@ {%- endif -%} {%- endset -%} - {%- set ddl -%} - create schema if not exists {{ fqn }} - {%- endset -%} + {% set schema_exists_query %} + select * from {{ source_node.database }}.INFORMATION_SCHEMA.SCHEMATA where schema_name = '{{ source_node.schema }}' limit 1 + {% endset %} + {% if execute %} + {% set schema_exists = run_query(schema_exists_query)|length > 0 %} + {% else %} + {% set schema_exists = false %} + {% endif %} + + {%- if not schema_exists -%} + {%- set ddl -%} + create schema if not exists {{ fqn }} + {%- endset -%} + {%- endif -%} {{ return(ddl) }} {%- endmacro -%} diff --git a/run_test.sh b/run_test.sh index 903c0210..2b6c99a7 100755 --- a/run_test.sh +++ b/run_test.sh @@ -1,6 +1,6 @@ #!/bin/bash -echo "Setting up virtual environment" +echo "Setting up virtual environment for dbt-$1" VENV="venv/bin/activate" if [[ ! -f $VENV ]]; then From e523ef630a2983ed72be74f8da09db57eab96567 Mon Sep 17 00:00:00 2001 From: Thomas van Latum Date: Wed, 21 Jun 2023 22:32:34 +0200 Subject: [PATCH 2/2] fix when we do not need to update the schema --- macros/plugins/bigquery/create_external_schema.sql | 7 ++++--- .../plugins/bigquery/get_external_build_plan.sql | 14 ++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/macros/plugins/bigquery/create_external_schema.sql b/macros/plugins/bigquery/create_external_schema.sql index 1199dfb0..07c00b95 100644 --- a/macros/plugins/bigquery/create_external_schema.sql +++ b/macros/plugins/bigquery/create_external_schema.sql @@ -20,7 +20,8 @@ {%- set ddl -%} create schema if not exists {{ fqn }} {%- endset -%} - {%- endif -%} - - {{ return(ddl) }} + {{ return(ddl) }} + {%- else -%} + {{ return('') }} + {% endif %} {%- endmacro -%} diff --git a/macros/plugins/bigquery/get_external_build_plan.sql b/macros/plugins/bigquery/get_external_build_plan.sql index f90c1bfa..b1ce0acd 100644 --- a/macros/plugins/bigquery/get_external_build_plan.sql +++ b/macros/plugins/bigquery/get_external_build_plan.sql @@ -11,10 +11,16 @@ {% set create_or_replace = (old_relation is none or var('ext_full_refresh', false)) %} {% if create_or_replace %} - {% set build_plan = build_plan + [ - dbt_external_tables.create_external_schema(source_node), - dbt_external_tables.create_external_table(source_node) - ] %} + {% if not dbt_external_tables.create_external_schema(source_node)|length %} + {% set build_plan = build_plan + [ + dbt_external_tables.create_external_table(source_node) + ] %} + {% else %} + {% set build_plan = build_plan + [ + dbt_external_tables.create_external_schema(source_node), + dbt_external_tables.create_external_table(source_node) + ] %} + {% endif %} {% else %} {% set build_plan = build_plan + dbt_external_tables.refresh_external_table(source_node) %} {% endif %}