From 9be7f8c76efe9cb1bd206f1f3a167ed287655364 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:17:07 +0100 Subject: [PATCH 1/5] Updating package dependendcies --- packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.yml b/packages.yml index dbfadab..a6cd29d 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,5 @@ packages: - package: fivetran/fivetran_utils - version: [">=0.1.0", "<0.2.0"] + version: [">=0.2.0", "<0.3.0"] - package: fishtown-analytics/spark_utils version: [">=0.2.0", "<0.3.0"] From 682dc8c4dd38b819570edd06cf2c5d9266fee134 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:17:08 +0100 Subject: [PATCH 2/5] Updating require-dbt-version --- dbt_project.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 03bf311..722878e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,13 +1,10 @@ name: 'twitter_ads_source' -version: '0.2.3' +version: '0.3.0' config-version: 2 - -require-dbt-version: [">=0.18.0", "<0.20.0"] - +require-dbt-version: [">=0.20.0"] models: twitter_ads_source: +schema: stg_twitter_ads - vars: twitter_ads_source: account_history: "{{ source('twitter_ads','account_history') }}" @@ -17,4 +14,4 @@ vars: promoted_tweet_report: "{{ source('twitter_ads','promoted_tweet_report') }}" tweet_url: "{{ source('twitter_ads','tweet_url') }}" dbt_utils_dispatch_list: ['spark_utils', 'fivetran_utils'] - fivetran_utils_dispatch_list: ['spark_utils'] \ No newline at end of file + fivetran_utils_dispatch_list: ['spark_utils'] From 0b2ff87d7879b3b625881f8f5a96069ddee97190 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:17:09 +0100 Subject: [PATCH 3/5] Updating dbt version in requirements.txt --- integration_tests/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index b0df5fe..b06ff52 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.19.1 -dbt-spark==0.19.1 -dbt-spark[PyHive]==0.19.1 +dbt==0.20.0 +dbt-spark==0.20.0 +dbt-spark[PyHive]==0.20.0 From 222a78b5c77e77dc02b28adac92f93a88b762466 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:17:52 +0100 Subject: [PATCH 4/5] Update dbt_project.yml --- integration_tests/dbt_project.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index e884817..8cf89d5 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -20,4 +20,8 @@ seeds: tweet_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" twitter_promoted_tweet_history_data: +column_types: - tweet_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" \ No newline at end of file + tweet_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] From 525118345ee5fde29b56eec9d6af35522ca9e02b Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Fri, 23 Jul 2021 21:10:05 -0500 Subject: [PATCH 5/5] dbt 20 readme edits --- README.md | 23 ++++++++++++++++++++--- dbt_project.yml | 2 -- integration_tests/dbt_project.yml | 2 +- packages.yml | 8 ++++---- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 1faffcd..748582c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt logo and version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange) # Twitter Ads (Source) This package models Twitter Ads data from [Fivetran's connector](https://fivetran.com/docs/applications/twitter-ads). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/twitter-ads#schemainformation). @@ -12,9 +13,13 @@ This package contains staging models, designed to work simultaneously with our [ ## Installation Instructions Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions, or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. -## Database Support +Include in your `packages.yml` -This package has been tested on BigQuery, Snowflake, Redshift, Postgres, and Databricks. +```yaml +packages: + - package: fivetran/twitter_ads_source + version: [">=0.3.0", "<0.4.0"] +``` ## Configuration By default, this package will look for your Twitter Ads data in the `twitter_ads` schema of your [target database](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile). If this is not where your Twitter Ads data is, please add the following configuration to your `dbt_project.yml` file: @@ -29,7 +34,6 @@ vars: twitter_ads_schema: your_schema_name twitter_ads_database: your_database_name ``` - ### Changing the Build Schema By default this package will build the Twitter Ads staging models within a schema titled ( + `_stg_twitter_ads`) in your target database. If this is not where you would like your Twitter Ads staging data to be written to, add the following configuration to your `dbt_project.yml` file: @@ -41,6 +45,19 @@ models: twitter_ads_source: +schema: my_new_schema_name # leave blank for just the target_schema ``` +## Database Support + +This package has been tested on BigQuery, Snowflake, Redshift, Postgres, and Databricks. + +### Databricks Dispatch Configuration +dbt `v0.20.0` introduced a new project-level dispatch configuration that enables an "override" setting for all dispatched macros. If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively. +```yml +# dbt_project.yml + +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] +``` ## Contributions diff --git a/dbt_project.yml b/dbt_project.yml index 722878e..4ed9f7c 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -13,5 +13,3 @@ vars: promoted_tweet_history: "{{ source('twitter_ads','promoted_tweet_history') }}" promoted_tweet_report: "{{ source('twitter_ads','promoted_tweet_report') }}" tweet_url: "{{ source('twitter_ads','tweet_url') }}" - dbt_utils_dispatch_list: ['spark_utils', 'fivetran_utils'] - fivetran_utils_dispatch_list: ['spark_utils'] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 8cf89d5..2223eef 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'twitter_source_integration_tests' -version: '0.2.3' +version: '0.3.0' profile: 'integration_tests' config-version: 2 snapshot-paths: ["snapshots"] diff --git a/packages.yml b/packages.yml index a6cd29d..59675cf 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,5 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.2.0", "<0.3.0"] -- package: fishtown-analytics/spark_utils - version: [">=0.2.0", "<0.3.0"] + - package: fivetran/fivetran_utils + version: [">=0.2.0", "<0.3.0"] + - package: dbt-labs/spark_utils + version: [">=0.2.0", "<0.3.0"]