Updates for dbt-utils to dbt-core cross-db macro migration

.buildkite/scripts/run_models.sh

@@ -19,5 +19,5 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{twitter_ads__using_keywords: False}' --target "$db" --full-refresh
-dbt test --target "$db"
+dbt run --vars '{twitter_ads__using_keywords: false, ad_reporting__url_report__using_null_filter: false}' --target "$db" --full-refresh
+dbt test --vars '{twitter_ads__using_keywords: false, ad_reporting__url_report__using_null_filter: false}' --target "$db"

CHANGELOG.md

@@ -1,3 +1,47 @@
+# dbt_twitter v0.6.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #14](https://github.com/fivetran/dbt_twitter/pull/14) includes the following breaking changes:
+- Dispatch update for dbt-utils to dbt-core cross-db macros migration. Specifically `{{ dbt_utils.<macro> }}` have been updated to `{{ dbt.<macro> }}` for the below macros:
+    - `any_value`
+    - `bool_or`
+    - `cast_bool_to_text`
+    - `concat`
+    - `date_trunc`
+    - `dateadd`
+    - `datediff`
+    - `escape_single_quotes`
+    - `except`
+    - `hash`
+    - `intersect`
+    - `last_day`
+    - `length`
+    - `listagg`
+    - `position`
+    - `replace`
+    - `right`
+    - `safe_cast`
+    - `split_part`
+    - `string_literal`
+    - `type_bigint`
+    - `type_float`
+    - `type_int`
+    - `type_numeric`
+    - `type_string`
+    - `type_timestamp`
+    - `array_append`
+    - `array_concat`
+    - `array_construct`
+- For `current_timestamp` and `current_timestamp_in_utc` macros, the dispatch AND the macro names have been updated to the below, respectively:
+    - `dbt.current_timestamp_backcompat`
+    - `dbt.current_timestamp_in_utc_backcompat`
+- Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
+
+## 🎉 Features 🎉
+- For use in the [dbt_ad_reporting package](https://github.com/fivetran/dbt_ad_reporting), users can now allow records having nulls in url fields to be included in the `ad_reporting__url_report` model. See the [dbt_ad_reporting README](https://github.com/fivetran/dbt_ad_reporting) for more details. [#17](https://github.com/fivetran/dbt_twitter/pull/17) 
+## 🚘 Under the Hood 🚘
+- Disabled the `not_null` test for `twitter_ads__url_report` when null urls are allowed. [#17](https://github.com/fivetran/dbt_twitter/pull/17)
+
 # dbt_twitter v0.5.1
 ## Fixes
 - Fix the package name in the README ([#16](https://github.com/fivetran/dbt_twitter/pull/16))

README.md

@@ -3,7 +3,7 @@
         href="https://github.com/fivetran/dbt_twitter/blob/main/LICENSE">
         <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
     <a alt="dbt-core">
-        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.0.0_<2.0.0-orange.svg" /></a>
+        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.3.0_<2.0.0-orange.svg" /></a>
     <a alt="Maintained?">
         <img src="https://img.shields.io/badge/Maintained%3F-yes-green.svg" /></a>
     <a alt="PRs">
@@ -52,7 +52,7 @@ Include the following Twitter package version in your `packages.yml` file:
 # packages.yml
 packages:
   - package: fivetran/twitter_ads
-    version: [">=0.5.0", "<0.6.0"]
+    version: [">=0.6.0", "<0.7.0"]
 ```
 
 ## Step 3: Define database and schema variables
@@ -133,11 +133,13 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/twitter_source
-      version: [">=0.5.0", "<0.6.0"]
+      version: [">=0.6.0", "<0.7.0"]
     - package: fivetran/fivetran_utils
-      version: [">=0.3.0", "<0.4.0"]
+      version: [">=0.4.0", "<0.5.0"]
     - package: dbt-labs/dbt_utils
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=1.0.0", "<2.0.0"]
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
 ```
 
 # 🙌 How is this package maintained and can I contribute?

dbt_project.yml

@@ -1,13 +1,11 @@
 name: 'twitter_ads'
-version: '0.5.1'
+version: '0.6.0'
 config-version: 2
-require-dbt-version: [">=1.0.0", "<2.0.0"]
-
+require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   twitter_ads:
     +schema: twitter_ads
     +materialized: table
-
 vars:
   twitter_ads:
     line_item_history: "{{ ref('stg_twitter_ads__line_item_history') }}"
@@ -20,9 +18,7 @@ vars:
     campaign_report: "{{ ref('stg_twitter_ads__campaign_report') }}"
     line_item_report: "{{ ref('stg_twitter_ads__line_item_report') }}"
     line_item_keywords_report: "{{ ref('stg_twitter_ads__line_item_keywords_report') }}"
-
     twitter_ads__campaign_report_passthrough_metrics: []
     twitter_ads__line_item_report_passthrough_metrics: []
     twitter_ads__line_item_keywords_report_passthrough_metrics: []
     twitter_ads__promoted_tweet_report_passthrough_metrics: []
-

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: twitter_ads_integration_tests
+      schema: twitter_ads_integration_tests_3
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: twitter_ads_integration_tests
+      schema: twitter_ads_integration_tests_3
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: twitter_ads_integration_tests
+      schema: twitter_ads_integration_tests_3
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: twitter_ads_integration_tests
+      schema: twitter_ads_integration_tests_3
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: twitter_ads_integration_tests
+      schema: twitter_ads_integration_tests_3
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,12 +1,12 @@
 name: 'twitter_integration_tests'
-version: '0.5.1'
+version: '0.6.0'
 profile: 'integration_tests'
 config-version: 2
 snapshot-paths: ["snapshots"]
 
 vars:
   twitter_ads_source:
-    twitter_ads_schema: twitter_ads_integration_tests
+    twitter_ads_schema: twitter_ads_integration_tests_3
 
     twitter_ads_account_history_identifier: "twitter_account_history_data"
     twitter_ads_campaign_history_identifier: "twitter_campaign_history_data"

integration_tests/requirements.txt

@@ -1,7 +1,7 @@
-dbt-snowflake>=1.0.0,<1.3.0
-dbt-bigquery>=1.0.0,<1.3.0
-dbt-redshift>=1.0.0,<1.3.0
-dbt-postgres>=1.0.0,<1.3.0
-dbt-spark>=1.0.0,<1.3.0
-dbt-spark[PyHive]>=1.0.0,<1.3.0
-dbt-databricks>=1.0.0,<1.3.0
+dbt-snowflake>=1.3.0,<2.0.0
+dbt-bigquery>=1.3.0,<2.0.0
+dbt-redshift>=1.3.0,<2.0.0
+dbt-postgres>=1.3.0,<2.0.0
+dbt-spark>=1.3.0,<2.0.0
+dbt-spark[PyHive]>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0

models/twitter.yml

@@ -338,6 +338,9 @@ models:
             - line_item_id
             - campaign_id 
             - account_id
+          config:
+            # Below is so we don't run this test if the user has allowed urls to have null entries
+            enabled: "{{ var('ad_reporting__url_report__using_null_filter', True) }}"
     columns:
       - name: date_day
         description: The date of the performance.
@@ -371,6 +374,11 @@ models:
         description: The full text of the tweet that the URL appeared in.
       - name: base_url
         description: The base URL of the ad, extracted from the `expanded_url`.
+        tests:
+          - not_null:
+              config:
+                # Below is so we don't run this test if the user has allowed urls to have null entries
+                enabled: "{{ var('ad_reporting__url_report__using_null_filter', True) }}"
       - name: url_host
         description: The URL host of the ad, extracted from the `expanded_url`.
       - name: url_path

models/twitter_ads__url_report.sql

@@ -95,8 +95,10 @@ final as (
     left join accounts
         on report.account_id = accounts.account_id
 
-    where tweet_url.expanded_url is not null
-
+    {% if var('ad_reporting__url_report__using_null_filter', True) %}
+        where tweet_url.expanded_url is not null
+    {% endif %}
+
     {{ dbt_utils.group_by(n=23) }}
 
 

packages.yml

@@ -1,3 +1,3 @@
 packages:
-  - package: fivetran/twitter_ads_source
-    version: [">=0.5.0", "<0.6.0"]
+- package: fivetran/twitter_ads_source
+  version: [">=0.6.0", "<0.7.0"]