diff --git a/README.md b/README.md index 1faffcd..748582c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt logo and version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange) # Twitter Ads (Source) This package models Twitter Ads data from [Fivetran's connector](https://fivetran.com/docs/applications/twitter-ads). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/twitter-ads#schemainformation). @@ -12,9 +13,13 @@ This package contains staging models, designed to work simultaneously with our [ ## Installation Instructions Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions, or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. -## Database Support +Include in your `packages.yml` -This package has been tested on BigQuery, Snowflake, Redshift, Postgres, and Databricks. +```yaml +packages: + - package: fivetran/twitter_ads_source + version: [">=0.3.0", "<0.4.0"] +``` ## Configuration By default, this package will look for your Twitter Ads data in the `twitter_ads` schema of your [target database](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile). If this is not where your Twitter Ads data is, please add the following configuration to your `dbt_project.yml` file: @@ -29,7 +34,6 @@ vars: twitter_ads_schema: your_schema_name twitter_ads_database: your_database_name ``` - ### Changing the Build Schema By default this package will build the Twitter Ads staging models within a schema titled ( + `_stg_twitter_ads`) in your target database. If this is not where you would like your Twitter Ads staging data to be written to, add the following configuration to your `dbt_project.yml` file: @@ -41,6 +45,19 @@ models: twitter_ads_source: +schema: my_new_schema_name # leave blank for just the target_schema ``` +## Database Support + +This package has been tested on BigQuery, Snowflake, Redshift, Postgres, and Databricks. + +### Databricks Dispatch Configuration +dbt `v0.20.0` introduced a new project-level dispatch configuration that enables an "override" setting for all dispatched macros. If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively. +```yml +# dbt_project.yml + +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] +``` ## Contributions diff --git a/dbt_project.yml b/dbt_project.yml index 03bf311..4ed9f7c 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,13 +1,10 @@ name: 'twitter_ads_source' -version: '0.2.3' +version: '0.3.0' config-version: 2 - -require-dbt-version: [">=0.18.0", "<0.20.0"] - +require-dbt-version: [">=0.20.0"] models: twitter_ads_source: +schema: stg_twitter_ads - vars: twitter_ads_source: account_history: "{{ source('twitter_ads','account_history') }}" @@ -16,5 +13,3 @@ vars: promoted_tweet_history: "{{ source('twitter_ads','promoted_tweet_history') }}" promoted_tweet_report: "{{ source('twitter_ads','promoted_tweet_report') }}" tweet_url: "{{ source('twitter_ads','tweet_url') }}" - dbt_utils_dispatch_list: ['spark_utils', 'fivetran_utils'] - fivetran_utils_dispatch_list: ['spark_utils'] \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index e884817..2223eef 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'twitter_source_integration_tests' -version: '0.2.3' +version: '0.3.0' profile: 'integration_tests' config-version: 2 snapshot-paths: ["snapshots"] @@ -20,4 +20,8 @@ seeds: tweet_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" twitter_promoted_tweet_history_data: +column_types: - tweet_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" \ No newline at end of file + tweet_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index b0df5fe..b06ff52 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.19.1 -dbt-spark==0.19.1 -dbt-spark[PyHive]==0.19.1 +dbt==0.20.0 +dbt-spark==0.20.0 +dbt-spark[PyHive]==0.20.0 diff --git a/packages.yml b/packages.yml index dbfadab..59675cf 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,5 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.1.0", "<0.2.0"] -- package: fishtown-analytics/spark_utils - version: [">=0.2.0", "<0.3.0"] + - package: fivetran/fivetran_utils + version: [">=0.2.0", "<0.3.0"] + - package: dbt-labs/spark_utils + version: [">=0.2.0", "<0.3.0"]