Merge pull request #267 from fishtown-analytics/feature/0-18-adapter-…

…dispatch Feature: use adapter.dispatch
dbt-labs · Aug 21, 2020 · 5f396cc · 5f396cc
2 parents f13f9b5 + 9760ba1
commit 5f396cc
Show file tree

Hide file tree

Showing 31 changed files with 131 additions and 76 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,12 @@
 
 ## Features
 
+* Switch usage of `adapter_macro` to `adapter.dispatch`, and define `dbt_utils_dispatch_list`,
+enabling users of community-supported database plugins to add or override macro implementations
+specific to their database (#267)
+* Use `add_ephemeral_prefix` instead of hard-coding a string literal, to support
+database adapters that use different prefixes (#267)
+
 ## Quality of life
 
 # dbt-utils v0.5.1

diff --git a/README.md b/README.md
@@ -777,6 +777,42 @@ We welcome contributions to this repo! To contribute a new feature or a fix, ple
 
 ----
 
+### Dispatch macros
+
+**Note:** This is primarily relevant to users and maintainers of community-supported
+database plugins. If you use Postgres, Redshift, Snowflake, or Bigquery, this likely
+does not apply to you.
+
+dbt v0.18.0 introduces `adapter.dispatch()`, a reliable way to define different implementations of the same macro
+across different databases.
+
+All dispatched macros in `dbt_utils` have an override setting: a `var` named
+`dbt_utils_dispatch_list` that accepts a list of package names. If you set this 
+variable in your project, when dbt searches for implementations of a dispatched
+`dbt_utils` macro, it will search through your listed packages _before_ using
+the implementations defined in `dbt_utils`.
+
+Set the variable:
+```yml
+vars:
+  dbt_utils_dispatch_list:
+    - first_package_to_search    # likely the name of your root project
+    - second_package_to_search   # likely an "add-on" package, such as spark_utils
+    # dbt_utils is always the last place searched
+```
+
+When running on Spark, if dbt needs to dispatch `dbt_utils.datediff`, it will search for the following in order:
+```
+first_package_to_search.spark__datediff
+first_package_to_search.default__datediff
+second_package_to_search.spark__datediff
+second_package_to_search.default__datediff
+dbt_utils.spark__datediff
+dbt_utils.default__datediff
+```
+
+----
+
 ### Getting started with dbt
 
 - [What is dbt]?

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 name: 'dbt_utils'
 version: '0.1.0'
 
-require-dbt-version: [">=0.17.0", "<0.18.0"]
+require-dbt-version: [">=0.18.0", "<0.19.0"]
 config-version: 2
 
 target-path: "target"

diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql
@@ -9,8 +9,7 @@
 with date_spine as (
 
     {% if target.type == 'postgres' %}
-        {{ log("WARNING: Not testing - datediff macro is unsupported on Postgres", info=True) }}
-        select * from {{ ref('data_date_spine') }}
+        {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }}
 
     {% elif target.type == 'bigquery' %}
         select cast(date_day as date) as date_day

diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql
@@ -1,4 +1,4 @@
 {{ config(materialized = 'table') }}
 
-{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_') %}
+{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_%') %}
 {{ dbt_utils.union_relations(relations) }}
diff --git a/macros/cross_db_utils/_get_utils_namespaces.sql b/macros/cross_db_utils/_get_utils_namespaces.sql
@@ -0,0 +1,4 @@
+{% macro _get_utils_namespaces() %}
+  {% set override_namespaces = var('dbt_utils_dispatch_list', []) %}
+  {% do return(override_namespaces + ['dbt_utils']) %}
+{% endmacro %}
diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql
@@ -1,7 +1,8 @@
 {% macro _is_ephemeral(obj, macro) %}
     {%- if obj.is_cte -%}
-        {% if obj.name.startswith('__dbt__CTE__') %}
-            {% set model_name = obj.name[12:] %}
+        {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %}
+        {% if obj.name.startswith(ephemeral_prefix) %}
+            {% set model_name = obj.name[(ephemeral_prefix|length):] %}
         {% else %}
             {% set model_name = obj.name %}
         {%- endif -%}

diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql
@@ -1,23 +1,23 @@
+
+
 {% macro concat(fields) -%}
-  {{ adapter_macro('dbt_utils.concat', fields) }}
+  {{ adapter.dispatch('concat', packages = dbt_utils._get_utils_namespaces())(fields) }}
 {%- endmacro %}
 
-
 {% macro default__concat(fields) -%}
     concat({{ fields|join(', ') }})
 {%- endmacro %}
 
-
 {% macro alternative_concat(fields) %}
     {{ fields|join(' || ') }}
 {% endmacro %}
 
 
 {% macro redshift__concat(fields) %}
-    {{dbt_utils.alternative_concat(fields)}}
+    {{ dbt_utils.alternative_concat(fields) }}
 {% endmacro %}
 
 
 {% macro snowflake__concat(fields) %}
-    {{dbt_utils.alternative_concat(fields)}}
+    {{ dbt_utils.alternative_concat(fields) }}
 {% endmacro %}
diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql
@@ -1,5 +1,5 @@
 {% macro current_timestamp() -%}
-  {{ adapter_macro('dbt_utils.current_timestamp') }}
+  {{ adapter.dispatch('current_timestamp', packages = dbt_utils._get_utils_namespaces())() }}
 {%- endmacro %}
 
 {% macro default__current_timestamp() %}
@@ -17,7 +17,7 @@
 
 
 {% macro current_timestamp_in_utc() -%}
-  {{ adapter_macro('dbt_utils.current_timestamp_in_utc') }}
+  {{ adapter.dispatch('current_timestamp_in_utc', packages = dbt_utils._get_utils_namespaces())() }}
 {%- endmacro %}
 
 {% macro default__current_timestamp_in_utc() %}

diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql
@@ -1,7 +1,7 @@
 {# string  -------------------------------------------------     #}
 
 {%- macro type_string() -%}
-  {{ adapter_macro('dbt_utils.type_string') }}
+  {{ adapter.dispatch('type_string', packages = dbt_utils._get_utils_namespaces())() }}
 {%- endmacro -%}
 
 {% macro default__type_string() %}
@@ -25,7 +25,7 @@
 {# timestamp  -------------------------------------------------     #}
 
 {%- macro type_timestamp() -%}
-  {{ adapter_macro('dbt_utils.type_timestamp') }}
+  {{ adapter.dispatch('type_timestamp', packages = dbt_utils._get_utils_namespaces())() }}
 {%- endmacro -%}
 
 {% macro default__type_timestamp() %}
@@ -40,7 +40,7 @@
 {# float  -------------------------------------------------     #}
 
 {%- macro type_float() -%}
-  {{ adapter_macro('dbt_utils.type_float') }}
+  {{ adapter.dispatch('type_float', packages = dbt_utils._get_utils_namespaces())() }}
 {%- endmacro -%}
 
 {% macro default__type_float() %}
@@ -54,7 +54,7 @@
 {# numeric  ------------------------------------------------     #}
 
 {%- macro type_numeric() -%}
-  {{ adapter_macro('dbt_utils.type_numeric') }}
+  {{ adapter.dispatch('type_numeric', packages = dbt_utils._get_utils_namespaces())() }}
 {%- endmacro -%}
 
 {% macro default__type_numeric() %}
@@ -69,7 +69,7 @@
 {# bigint  -------------------------------------------------     #}
 
 {%- macro type_bigint() -%}
-  {{ adapter_macro('dbt_utils.type_bigint') }}
+  {{ adapter.dispatch('type_bigint', packages = dbt_utils._get_utils_namespaces())() }}
 {%- endmacro -%}
 
 {% macro default__type_bigint() %}
@@ -83,7 +83,7 @@
 {# int  -------------------------------------------------     #}
 
 {%- macro type_int() -%}
-  {{ adapter_macro('dbt_utils.type_int') }}
+  {{ adapter.dispatch('type_int', packages = dbt_utils._get_utils_namespaces())() }}
 {%- endmacro -%}
 
 {% macro default__type_int() %}

diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql
@@ -1,5 +1,5 @@
 {% macro date_trunc(datepart, date) -%}
-  {{ adapter_macro('dbt_utils.date_trunc', datepart, date) }}
+  {{ adapter.dispatch('date_trunc', packages = dbt_utils._get_utils_namespaces()) (datepart, date) }}
 {%- endmacro %}
 
 {% macro default__date_trunc(datepart, date) %}

diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql
@@ -1,5 +1,5 @@
 {% macro dateadd(datepart, interval, from_date_or_timestamp) %}
-  {{ adapter_macro('dbt_utils.dateadd', datepart, interval, from_date_or_timestamp) }}
+  {{ adapter.dispatch('dateadd', packages = dbt_utils._get_utils_namespaces())(datepart, interval, from_date_or_timestamp) }}
 {% endmacro %}
 
 
@@ -23,7 +23,6 @@
 
 {% endmacro %}
 
-
 {% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}
 
     {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))

diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql
@@ -1,5 +1,5 @@
 {% macro datediff(first_date, second_date, datepart) %}
-  {{ adapter_macro('dbt_utils.datediff', first_date, second_date, datepart) }}
+  {{ adapter.dispatch('datediff', packages = dbt_utils._get_utils_namespaces())(first_date, second_date, datepart) }}
 {% endmacro %}
 
 
@@ -24,7 +24,6 @@
 
 {% endmacro %}
 
-
 {% macro postgres__datediff(first_date, second_date, datepart) %}
 
     {% if datepart == 'year' %}

diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql
@@ -1,5 +1,5 @@
 {% macro except() %}
-  {{ adapter_macro('dbt_utils.except') }}
+  {{ adapter.dispatch('except', packages = dbt_utils._get_utils_namespaces())() }}
 {% endmacro %}
 
 

diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql
@@ -1,5 +1,5 @@
 {% macro hash(field) -%}
-  {{ adapter_macro('dbt_utils.hash', field) }}
+  {{ adapter.dispatch('hash', packages = dbt_utils._get_utils_namespaces()) (field) }}
 {%- endmacro %}
 
 

diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql
@@ -4,7 +4,7 @@
     Use `adapter.quote` instead. The {}.{} model triggered this warning. \
     '.format(model.package_name, model.name) -%}
   {%- do exceptions.warn(error_message) -%}
-  {{ adapter_macro('dbt_utils.identifier', value) }}	
+  {{ adapter.dispatch('identifier', packages = dbt_utils._get_utils_namespaces()) (value) }}
 {% endmacro %}	
 
 {% macro default__identifier(value) -%}	

diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql
@@ -1,5 +1,5 @@
 {% macro intersect() %}
-  {{ adapter_macro('dbt_utils.intersect') }}
+  {{ adapter.dispatch('intersect', packages = dbt_utils._get_utils_namespaces())() }}
 {% endmacro %}
 
 

diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql
@@ -4,7 +4,7 @@ testing is required to validate that it will work on other dateparts.
 */
 
 {% macro last_day(date, datepart) %}
-  {{ adapter_macro('dbt_utils.last_day', date, datepart) }}
+  {{ adapter.dispatch('last_day', packages = dbt_utils._get_utils_namespaces()) (date, datepart) }}
 {% endmacro %}
 
 

diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql
@@ -1,5 +1,5 @@
 {% macro length(expression) -%}
-    {{ adapter_macro('dbt_utils.length', expression) }}
+    {{ adapter.dispatch('length', packages = dbt_utils._get_utils_namespaces()) (expression) }}
 {% endmacro %}
 
 

diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql
@@ -1,6 +1,6 @@
 
 {%- macro string_literal(value) -%}
-  {{ adapter_macro('dbt_utils.string_literal', value) }}
+  {{ adapter.dispatch('string_literal', packages = dbt_utils._get_utils_namespaces()) (value) }}
 {%- endmacro -%}
 
 {% macro default__string_literal(value) -%}

diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql
@@ -1,5 +1,5 @@
 {% macro position(substring_text, string_text) -%}
-    {{ adapter_macro('dbt_utils.position', substring_text, string_text) }}
+    {{ adapter.dispatch('position', packages = dbt_utils._get_utils_namespaces()) (substring_text, string_text) }}
 {% endmacro %}
 
 

diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql
@@ -1,5 +1,5 @@
 {% macro replace(field, old_chars, new_chars) -%}
-    {{ adapter_macro('dbt_utils.replace', field, old_chars, new_chars) }}
+    {{ adapter.dispatch('replace', packages = dbt_utils._get_utils_namespaces()) (field, old_chars, new_chars) }}
 {% endmacro %}
 
 

diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql
@@ -1,5 +1,5 @@
 {% macro right(string_text, length_expression) -%}
-    {{ adapter_macro('dbt_utils.right', string_text, length_expression) }}
+    {{ adapter.dispatch('right', packages = dbt_utils._get_utils_namespaces()) (string_text, length_expression) }}
 {% endmacro %}
 
 {% macro default__right(string_text, length_expression) %}

diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql
@@ -1,5 +1,5 @@
 {% macro safe_cast(field, type) %}
-  {{ adapter_macro('dbt_utils.safe_cast', field, type) }}
+  {{ adapter.dispatch('safe_cast', packages = dbt_utils._get_utils_namespaces()) (field, type) }}
 {% endmacro %}
 
 

diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql
@@ -1,5 +1,5 @@
 {% macro split_part(string_text, delimiter_text, part_number) %}
-  {{ adapter_macro('dbt_utils.split_part', string_text, delimiter_text, part_number) }}
+  {{ adapter.dispatch('split_part', packages = dbt_utils._get_utils_namespaces()) (string_text, delimiter_text, part_number) }}
 {% endmacro %}
 
 

diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql
@@ -1,5 +1,5 @@
 {% macro width_bucket(expr, min_value, max_value, num_buckets) %}
-  {{ adapter_macro('dbt_utils.width_bucket', expr, min_value, max_value, num_buckets) }}
+  {{ adapter.dispatch('width_bucket', packages = dbt_utils._get_utils_namespaces()) (expr, min_value, max_value, num_buckets) }}
 {% endmacro %}
 
 

diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql
@@ -2,7 +2,7 @@
 
     {%- call statement('get_tables', fetch_result=True) %}
 
-      {{ dbt_utils.get_tables_by_pattern(schema_pattern, table_pattern, exclude, database) }}
+      {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }}
 
     {%- endcall -%}
 
@@ -20,4 +20,16 @@
         {{ return([]) }}
     {%- endif -%}
 
-{% endmacro %}
+{% endmacro %}
+
+{% macro get_tables_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %}
+    {%- set error_message = '
+        Warning: the `get_tables_by_pattern` macro is no longer supported and will be deprecated in a future release of dbt-utils. \
+        Use the `get_relations_by_prefix` macro instead. \
+        The {}.{} model triggered this warning. \
+        '.format(model.package_name, model.name) -%}
+    {%- do exceptions.warn(error_message) -%}
+
+    {{ return(dbt_utils.get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database)) }}
+
+{% endmacro %} 
diff --git a/macros/sql/get_tables_by_pattern.sql b/macros/sql/get_tables_by_pattern.sql
diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql
@@ -0,0 +1,28 @@
+{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}
+    {{ adapter.dispatch('get_tables_by_pattern_sql', packages = dbt_utils._get_utils_namespaces())
+        (schema_pattern, table_pattern, exclude='', database=target.database) }}
+{% endmacro %}
+
+{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}
+
+        select distinct 
+            table_schema as "table_schema", table_name as "table_name"
+        from {{database}}.information_schema.tables
+        where table_schema ilike '{{ schema_pattern }}'
+        and table_name ilike '{{ table_pattern }}'
+        and table_name not ilike '{{ exclude }}'
+
+{% endmacro %}
+
+
+{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}
+
+        select distinct
+            table_schema, table_name
+
+        from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES
+        where table_schema = '{{schema_pattern}}'
+            and lower(table_name) like lower ('{{table_pattern}}')
+            and lower(table_name) not like lower ('{{exclude}}')
+
+{% endmacro %}