Lift + shift for cross-db macros

.circleci/config.yml

@@ -30,10 +30,16 @@ jobs:
             python3.8 -m venv venv
             . venv/bin/activate
             pip install --upgrade pip setuptools
-            pip install --pre --upgrade dbt-spark[ODBC]
+            pip install -r dev-requirements.txt
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
 
+      - run:
+          name: "Run Functional Tests"
+          command: |
+            . venv/bin/activate
+            python3 -m pytest tests/functional --profile databricks_sql_endpoint
+
       - run:
           name: "Run Tests - dbt-utils"
 

.gitignore

@@ -4,3 +4,5 @@
 /**/dbt_packages/
 /**/logs/
 /**/env/
+/**/__pycache__/
+test.env

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'spark_utils'
 version: '0.3.0'
 config-version: 2
-require-dbt-version: [">=1.0.0", "<2.0.0"]
+require-dbt-version: [">=1.2.0", "<2.0.0"]
 macro-paths: ["macros"]

dev-requirements.txt

@@ -0,0 +1,5 @@
+pytest
+pyodbc==4.0.32
+git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
+git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter
+git+https://github.com/dbt-labs/dbt-spark.git#egg=dbt-spark[ODBC]

integration_tests/dbt_utils/dbt_project.yml

@@ -30,6 +30,7 @@ seeds:
 models:
   dbt_utils_integration_tests:
     +file_format: delta
+
     sql:
       # macro doesn't work for this integration test (schema pattern)
       test_get_relations_by_pattern:
@@ -41,13 +42,19 @@ models:
       test_pivot_apostrophe:
         +enabled: false
     generic_tests:
-      # integration test doesn't work
+      # default version of this integration test uses an explicit cast to 'datetime'
+      # which SparkSQL does not support. override with our own version
       test_recency:
         +enabled: false
     cross_db_utils:
       # integration test doesn't work
       test_any_value:
         +enabled: false
-      # integration test doesn't work
-      test_listagg:
-        +enabled: false
+
+tests:
+  dbt_utils_integration_tests:
+    cross_db_utils:
+      # expect exactly two failures
+      # (both use "order by", which isn't supported in SparkSQL)
+      assert_equal_test_listagg_actual__expected:
+        error_if: ">2"

integration_tests/dbt_utils/models/test_recency.sql

@@ -0,0 +1,2 @@
+select
+    {{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as today

macros/dbt_utils/cross_db_utils/deprecated/assert_not_null.sql

@@ -0,0 +1,3 @@
+{% macro assert_not_null(function, arg) -%}
+  {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}
+{%- endmacro %}

macros/dbt_utils/cross_db_utils/deprecated/concat.sql

@@ -0,0 +1,3 @@
+{% macro spark__concat(fields) -%}
+    {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}
+{%- endmacro %}

macros/dbt_utils/cross_db_utils/datatypes.sql → macros/dbt_utils/cross_db_utils/deprecated/datatypes.sql

@@ -1,5 +1,5 @@
 {# numeric  ------------------------------------------------     #}
 
 {% macro spark__type_numeric() %}
-    decimal(28, 6)
+    {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}
 {% endmacro %}

macros/dbt_utils/cross_db_utils/deprecated/dateadd.sql

@@ -0,0 +1,6 @@
+{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}
+    -- dispatch here gets very very confusing
+    -- we just need to hint to dbt that this is a required macro for resolving dbt.spark__datediff()
+    -- {{ assert_not_null() }}
+    {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}
+{% endmacro %}

macros/dbt_utils/cross_db_utils/deprecated/datediff.sql

@@ -0,0 +1,6 @@
+{% macro spark__datediff(first_date, second_date, datepart) %}
+    -- dispatch here gets very very confusing
+    -- we just need to hint to dbt that this is a required macro for resolving dbt.spark__datediff()
+    -- {{ assert_not_null() }}
+    {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}
+{% endmacro %}

macros/dbt_utils/cross_db_utils/deprecated/split_part.sql

@@ -0,0 +1,3 @@
+{% macro spark__split_part(string_text, delimiter_text, part_number) %}
+  {{ return(adapter.dispatch('split_part', 'dbt')(string_text, delimiter_text, part_number)) }}
+{% endmacro %}

pytest.ini

@@ -0,0 +1,8 @@
+[pytest]
+filterwarnings =
+    ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning
+    ignore:unclosed file .*:ResourceWarning
+env_files =
+    test.env
+testpaths =
+    tests/functional