Merge pull request #3100 from prratek/specify-cols-to-update

Gets columns to update from config for BQ and Snowflake
dbt-labs · Mar 22, 2021 · 77c1071 · 77c1071
2 parents 934c23b + 48e367c
commit 77c1071
Show file tree

Hide file tree

Showing 9 changed files with 578 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@
 ### Features
 - Add optional configs for `require_partition_filter` and `partition_expiration_days` in BigQuery ([#1843](https://github.com/fishtown-analytics/dbt/issues/1843), [#2928](https://github.com/fishtown-analytics/dbt/pull/2928))
 - Fix for EOL SQL comments prevent entire line execution ([#2731](https://github.com/fishtown-analytics/dbt/issues/2731), [#2974](https://github.com/fishtown-analytics/dbt/pull/2974))
+- Add optional `merge_update_columns` config to specify columns to update for `merge` statements in BigQuery and Snowflake ([#1862](https://github.com/fishtown-analytics/dbt/issues/1862), [#3100](https://github.com/fishtown-analytics/dbt/pull/3100))
 - Use query comment JSON as job labels for BigQuery adapter when `query-comment.job-label` is set to `true` ([#2483](https://github.com/fishtown-analytics/dbt/issues/2483)), ([#3145](https://github.com/fishtown-analytics/dbt/pull/3145))
 
 ### Under the hood
@@ -29,6 +30,7 @@ Contributors:
 - [@VasiliiSurov](https://github.com/VasiliiSurov) ([#3104](https://github.com/fishtown-analytics/dbt/pull/3104))
 - [@jmcarp](https://github.com/jmcarp) ([#3145](https://github.com/fishtown-analytics/dbt/pull/3145))
 - [@bastienboutonnet](https://github.com/bastienboutonnet) ([#3151](https://github.com/fishtown-analytics/dbt/pull/3151))
+- [@prratek](https://github.com/prratek) ([#3100](https://github.com/fishtown-analytics/dbt/pull/3100))
 - [@techytushar](https://github.com/techytushar) ([#3158](https://github.com/fishtown-analytics/dbt/pull/3158))
 - [@cgopalan](https://github.com/cgopalan) ([#3165](https://github.com/fishtown-analytics/dbt/pull/3165))
 

diff --git a/core/dbt/include/global_project/macros/materializations/common/merge.sql b/core/dbt/include/global_project/macros/materializations/common/merge.sql
@@ -18,6 +18,7 @@
 {% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}
     {%- set predicates = [] if predicates is none else [] + predicates -%}
     {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
+    {%- set update_columns = config.get('merge_update_columns', default = dest_columns | map(attribute="quoted") | list) -%}
     {%- set sql_header = config.get('sql_header', none) -%}
 
     {% if unique_key %}
@@ -37,8 +38,8 @@
 
     {% if unique_key %}
     when matched then update set
-        {% for column in dest_columns -%}
-            {{ adapter.quote(column.name) }} = DBT_INTERNAL_SOURCE.{{ adapter.quote(column.name) }}
+        {% for column_name in update_columns -%}
+            {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}
             {%- if not loop.last %}, {%- endif %}
         {%- endfor %}
     {% endif %}

diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py
@@ -116,6 +116,7 @@ class BigqueryConfig(AdapterConfig):
     hours_to_expiration: Optional[int] = None
     require_partition_filter: Optional[bool] = None
     partition_expiration_days: Optional[int] = None
+    merge_update_columns: Optional[str] = None
 
 
 class BigQueryAdapter(BaseAdapter):

diff --git a/plugins/snowflake/dbt/adapters/snowflake/impl.py b/plugins/snowflake/dbt/adapters/snowflake/impl.py
@@ -26,6 +26,7 @@ class SnowflakeConfig(AdapterConfig):
     copy_grants: Optional[bool] = None
     snowflake_warehouse: Optional[str] = None
     query_tag: Optional[str] = None
+    merge_update_columns: Optional[str] = None
 
 
 class SnowflakeAdapter(SQLAdapter):

diff --git a/test/integration/001_simple_copy_test/models-merge-update/incremental_update_cols.sql b/test/integration/001_simple_copy_test/models-merge-update/incremental_update_cols.sql
@@ -0,0 +1,17 @@
+{{
+    config(
+        materialized = "incremental",
+        unique_key = "id",
+        merge_update_columns = ["email", "ip_address"]
+    )
+}}
+
+
+select *
+from {{ ref('seed') }}
+
+{% if is_incremental() %}
+
+    where load_date > (select max(load_date) from {{this}})
+
+{% endif %}
diff --git a/test/integration/001_simple_copy_test/seeds-merge-cols-initial/seed.csv b/test/integration/001_simple_copy_test/seeds-merge-cols-initial/seed.csv
@@ -0,0 +1,101 @@
+load_date,id,first_name,last_name,email,gender,ip_address
+2021-03-05,1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168
+2021-03-05,2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35
+2021-03-05,3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243
+2021-03-05,4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175
+2021-03-05,5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136
+2021-03-05,6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220
+2021-03-05,7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64
+2021-03-05,8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13
+2021-03-05,9,Gary,Day,gday8@nih.gov,Male,35.81.68.186
+2021-03-05,10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100
+2021-03-05,11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67
+2021-03-05,12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193
+2021-03-05,13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5
+2021-03-05,14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250
+2021-03-05,15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245
+2021-03-05,16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54
+2021-03-05,17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96
+2021-03-05,18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72
+2021-03-05,19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174
+2021-03-05,20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25
+2021-03-05,21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253
+2021-03-05,22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153
+2021-03-05,23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201
+2021-03-05,24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122
+2021-03-05,25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95
+2021-03-05,26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52
+2021-03-05,27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26
+2021-03-05,28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118
+2021-03-05,29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28
+2021-03-05,30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177
+2021-03-05,31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233
+2021-03-05,32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203
+2021-03-05,33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149
+2021-03-05,34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167
+2021-03-05,35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110
+2021-03-05,36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68
+2021-03-05,37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89
+2021-03-05,38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81
+2021-03-05,39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15
+2021-03-05,40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255
+2021-03-05,41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140
+2021-03-05,42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24
+2021-03-05,43,Sean,Mason,smason16@icq.com,Male,159.219.155.249
+2021-03-05,44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218
+2021-03-05,45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198
+2021-03-05,46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18
+2021-03-05,47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238
+2021-03-05,48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61
+2021-03-05,49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21
+2021-03-05,50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209
+2021-03-05,51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87
+2021-03-05,52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142
+2021-03-05,53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126
+2021-03-05,54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212
+2021-03-05,55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194
+2021-03-05,56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22
+2021-03-05,57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60
+2021-03-05,58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50
+2021-03-05,59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222
+2021-03-05,60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115
+2021-03-05,61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155
+2021-03-05,62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94
+2021-03-05,63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106
+2021-03-05,64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68
+2021-03-05,65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41
+2021-03-05,66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109
+2021-03-05,67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77
+2021-03-05,68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194
+2021-03-05,69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135
+2021-03-05,70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87
+2021-03-05,71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44
+2021-03-05,72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182
+2021-03-05,73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241
+2021-03-05,74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24
+2021-03-05,75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214
+2021-03-05,76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199
+2021-03-05,77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41
+2021-03-05,78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255
+2021-03-05,79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144
+2021-03-05,80,Rose,King,rking27@ucoz.com,Female,212.123.168.231
+2021-03-05,81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188
+2021-03-05,82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61
+2021-03-05,83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30
+2021-03-05,84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192
+2021-03-05,85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232
+2021-03-05,86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109
+2021-03-05,87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156
+2021-03-05,88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84
+2021-03-05,89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235
+2021-03-05,90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53
+2021-03-05,91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221
+2021-03-05,92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187
+2021-03-05,93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57
+2021-03-05,94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189
+2021-03-05,95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180
+2021-03-05,96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144
+2021-03-05,97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117
+2021-03-05,98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126
+2021-03-05,99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244
+2021-03-05,100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88