Merge pull request #16 from fivetran/MagicBot_1e3f282712

[MagicBot] Bumping package version

.circleci/config.yml

@@ -15,6 +15,7 @@ jobs:
       - run:
           name: "Setup dbt"
           command: |
+            sudo apt install libsasl2-dev
             python3 -m venv venv
             . venv/bin/activate
             pip install --upgrade pip setuptools
@@ -53,4 +54,4 @@ jobs:
             dbt deps
             dbt seed --target bigquery --full-refresh
             dbt run --target bigquery --full-refresh
-            dbt test --target bigquery
+            dbt test --target bigquery

.gitignore

@@ -2,3 +2,5 @@
 target/
 dbt_modules/
 logs/
+
+dbt_packages/

CHANGELOG.md

@@ -0,0 +1,14 @@
+# dbt_marketo v0.6.0
+🎉 dbt v1.0.0 Compatibility 🎉
+## 🚨 Breaking Changes 🚨
+- Adjusts the `require-dbt-version` to now be within the range [">=1.0.0", "<2.0.0"]. Additionally, the package has been updated for dbt v1.0.0 compatibility. If you are using a dbt version <1.0.0, you will need to upgrade in order to leverage the latest version of the package.
+  - For help upgrading your package, I recommend reviewing this GitHub repo's Release Notes on what changes have been implemented since your last upgrade.
+  - For help upgrading your dbt project to dbt v1.0.0, I recommend reviewing dbt-labs [upgrading to 1.0.0 docs](https://docs.getdbt.com/docs/guides/migration-guide/upgrading-to-1-0-0) for more details on what changes must be made.
+- Upgrades the package dependency to refer to the latest `dbt_marketo_source`. Additionally, the latest `dbt_marketo_source` package has a dependency on the latest `dbt_fivetran_utils`. Further, the latest `dbt_fivetran_utils` package also has a dependency on `dbt_utils` [">=0.8.0", "<0.9.0"].
+  - Please note, if you are installing a version of `dbt_utils` in your `packages.yml` that is not in the range above then you will encounter a package dependency error.
+
+## Under the Hood 
+- Redshift recently included pivot as a reserved word within the warehouse. As such, the `pivot` CTE within the `marketo__change_data_pivot` and `marketo__change_data_details` models have been changed to `pivots` to avoid the Redshift error.
+
+# dbt_marketo_source v0.1.0 -> v0.5.0
+Refer to the relevant release notes on the Github repository for specific details for the previous releases. Thank you!

README.md

@@ -1,4 +1,4 @@
-[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt Logo and Version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange)
+[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 # Marketo ([docs](https://fivetran-dbt-marketo.netlify.app/#!/overview))
 
 This package models Marketo data from [Fivetran's connector](https://fivetran.com/docs/applications/marketo). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/marketo#schema).
@@ -20,16 +20,14 @@ This package contains transformation models, designed to work simultaneously wit
 
 
 ## Installation Instructions
-`dbt_marketo` currently supports `dbt 0.20.x`.
-
 Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions, or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
 
 Include in your `packages.yml`
 
 ```yaml
 packages:
   - package: fivetran/marketo
-    version: [">=0.5.0", "<0.6.0"]
+    version: [">=0.6.0", "<0.7.0"]
 ```
 
 ## Configuration

dbt_project.yml

@@ -1,43 +1,35 @@
 name: 'marketo'
-version: '0.5.0'
+version: '0.6.0'
 config-version: 2
-require-dbt-version: ">=0.20.0"
+require-dbt-version: [">=1.0.0", "<2.0.0"]
 
 on-run-start: "{{ lead_history_columns_warning() }}"
 
 vars:
 
-    marketo__enable_programs: False
-    marketo__enable_campaigns: False
-
-    marketo:
-        lead_history_columns: [
-            'lead_status',
-            'urgency',
-            'priority',
-            'relative_score',
-            'relative_urgency', 
-            'demographic_score_marketing',
-            'behavior_score_marketing'
-        ]
+  marketo__enable_programs: false
+  marketo__enable_campaigns: false
 
-        activity_email_bounced: "{{ ref('stg_marketo__activity_email_bounced') }}"
-        activity_click_email: "{{ ref('stg_marketo__activity_click_email') }}"
-        activity_email_delivered: "{{ ref('stg_marketo__activity_email_delivered') }}"
-        activity_open_email: "{{ ref('stg_marketo__activity_open_email') }}"
-        activity_unsubscribe_email: "{{ ref('stg_marketo__activity_unsubscribe_email') }}"
-        activity_send_email: "{{ ref('stg_marketo__activity_send_email') }}"
+  marketo:
+    lead_history_columns: ['lead_status', 'urgency', 'priority', 'relative_score', 'relative_urgency', 'demographic_score_marketing', 'behavior_score_marketing']
 
-        change_data_value: "{{ ref('stg_marketo__activity_change_data_value') }}"
-        lead: "{{ ref('stg_marketo__lead') }}"
-        lead_describe: "{{ ref('stg_marketo__lead_describe') }}"
-        campaigns: "{{ ref('stg_marketo__campaigns') }}"
-        email_tempate_history: "{{ ref('stg_marketo__email_template_history') }}"
-        program: "{{ ref('stg_marketo__program') }}"
+    activity_email_bounced: "{{ ref('stg_marketo__activity_email_bounced') }}"
+    activity_click_email: "{{ ref('stg_marketo__activity_click_email') }}"
+    activity_email_delivered: "{{ ref('stg_marketo__activity_email_delivered') }}"
+    activity_open_email: "{{ ref('stg_marketo__activity_open_email') }}"
+    activity_unsubscribe_email: "{{ ref('stg_marketo__activity_unsubscribe_email') }}"
+    activity_send_email: "{{ ref('stg_marketo__activity_send_email') }}"
 
-models: 
-    marketo:
-        +materialized: table
-        +schema: marketo
-        intermediate:
-            +materialized: ephemeral
+    change_data_value: "{{ ref('stg_marketo__activity_change_data_value') }}"
+    lead: "{{ ref('stg_marketo__lead') }}"
+    lead_describe: "{{ ref('stg_marketo__lead_describe') }}"
+    campaigns: "{{ ref('stg_marketo__campaigns') }}"
+    email_tempate_history: "{{ ref('stg_marketo__email_template_history') }}"
+    program: "{{ ref('stg_marketo__program') }}"
+
+models:
+  marketo:
+    +materialized: table
+    +schema: marketo
+    intermediate:
+      +materialized: ephemeral

integration_tests/requirements.txt

@@ -1 +1,6 @@
-dbt~=0.20.0
+dbt-snowflake==1.0.0
+dbt-bigquery==1.0.0
+dbt-redshift==1.0.0
+dbt-postgres==1.0.0
+dbt-spark==1.0.0
+dbt-spark[PyHive]==1.0.0

models/intermediate/marketo__change_data_details.sql

@@ -35,7 +35,7 @@ with change_data as (
     left join lead_describe
         on change_data.primary_attribute_value_id = lead_describe.lead_describe_id
 
-), pivot as (
+), pivots as (
 
     -- For each column that is in both the lead_history_columns variable and the restname of the lead_describe table,
     -- find whether a change occurred for a given column on a given day for a given lead. 
@@ -59,9 +59,9 @@ with change_data as (
     select 
         *,
         {{ dbt_utils.surrogate_key(['lead_id','date_day'])}} as lead_day_id
-    from pivot
+    from pivots
 
 )
 
 select *
-from surrogate_key
+from surrogate_key

models/intermediate/marketo__change_data_pivot.sql

@@ -53,7 +53,7 @@ with change_data as (
     from event_order
     where row_num = 1
 
-), pivot as (
+), pivots as (
 
     -- For each column that is in both the lead_history_columns variable and the restname of the lead_describe table,
     -- pivot out the value into it's own column. This will feed the daily slowly changing dimension model.
@@ -76,9 +76,9 @@ with change_data as (
     select 
         *,
         {{ dbt_utils.surrogate_key(['lead_id','date_day'])}} as lead_day_id
-    from pivot
+    from pivots
 
 )
 
 select *
-from surrogate_key
+from surrogate_key

packages.yml

@@ -1,3 +1,3 @@
 packages:
-  - package: fivetran/marketo_source
-    version: [">=0.5.0", "<0.6.0"]
+- package: fivetran/marketo_source
+  version: [">=0.6.0", "<0.7.0"]