Merge pull request #5 from fivetran/feature/spark

Spark compatibility

.circleci/config.yml

@@ -15,12 +15,23 @@ jobs:
       - run:
           name: "Setup dbt"
           command: |
+            sudo apt install libsasl2-dev
             python3 -m venv venv
             . venv/bin/activate
             pip install --upgrade pip setuptools
             pip install -r integration_tests/requirements.txt
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
+      - run:
+          name: "Run Tests - Spark"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target spark --full-refresh
+            dbt run --target spark --full-refresh
+            dbt test --target spark
       - run:
           name: "Run Tests - Redshift"
           command: |

dbt_project.yml

@@ -16,4 +16,6 @@ vars:
     twitter_promoted_tweet_history: "{{ ref('stg_twitter_ads__promoted_tweet_history')}}"
     twitter_tweet_url: "{{ ref('stg_twitter_ads__tweet_url') }}"
     twitter_account_history: "{{ ref('stg_twitter_ads__account_history') }}"
-    twitter_campaign_history: "{{ ref('stg_twitter_ads__campaign_history') }}"
+    twitter_campaign_history: "{{ ref('stg_twitter_ads__campaign_history') }}"
+  dbt_utils_dispatch_list: ['spark_utils', 'fivetran_utils']
+  fivetran_utils_dispatch_list: ['spark_utils']

integration_tests/ci/sample.profiles.yml

@@ -34,4 +34,16 @@ integration_tests:
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
       schema: twitter_integration_tests
-      threads: 8
+      threads: 8
+    spark:
+      type: spark
+      method: http
+      schema: twitter_integration_tests
+      host: "{{ env_var('CI_SPARK_DBT_HOST') }}"
+      organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}"
+      token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}"
+      cluster: "{{ env_var('CI_SPARK_DBT_CLUSTER') }}"
+      port: 443
+      connect_timeout: 60
+      connect_retries: 5
+      threads: 4

integration_tests/requirements.txt

@@ -1 +1,3 @@
-dbt==0.19.0
+dbt==0.19.1
+dbt-spark==0.19.1
+dbt-spark[PyHive]==0.19.1

models/twitter__ad_adapter.sql

@@ -54,13 +54,13 @@ with accounts as (
         sum(metrics.impressions) as impressions
     from metrics
     left join promoted_tweet
-        using (promoted_tweet_id)
+        on metrics.promoted_tweet_id = promoted_tweet.promoted_tweet_id
     left join tweet_url
-        using (tweet_id)
+        on promoted_tweet.tweet_id = tweet_url.tweet_id
     left join line_items
-        using (line_item_id)
+        on promoted_tweet.line_item_id = line_items.line_item_id
     left join campaigns
-        using (campaign_id)
+        on line_items.campaign_id = campaigns.campaign_id
     {{ dbt_utils.group_by(13) }}
 
 ), unique_id as (

packages.yml

@@ -1,3 +1,3 @@
 packages:
   - package: fivetran/twitter_ads_source
-    version: [">=0.2.0","<0.3.0"]
+    version: [">=0.2.0","<0.3.0"]