From 11e23a0b7898790d096329e1e13e472fecb8bd11 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sat, 25 Mar 2017 13:51:35 -0400 Subject: [PATCH 1/3] wrap redshift DROP ... CASCADE in a lock --- dbt/adapters/redshift.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift.py index 81b3e344316..8f765d14fd3 100644 --- a/dbt/adapters/redshift.py +++ b/dbt/adapters/redshift.py @@ -1,7 +1,12 @@ +import multiprocessing + from dbt.adapters.postgres import PostgresAdapter from dbt.logger import GLOBAL_LOGGER as logger # noqa +drop_lock = multiprocessing.Lock() + + class RedshiftAdapter(PostgresAdapter): @classmethod @@ -42,3 +47,28 @@ def sort_qualifier(cls, sort_type, sort): return "{sort_type} sortkey({keys_csv})".format( sort_type=sort_type, keys_csv=keys_csv ) + + @classmethod + def drop(cls, profile, relation, relation_type, model_name=None): + global drop_lock + + to_return = None + + try: + drop_lock.acquire() + + connection = cls.get_connection(profile, model_name) + + cls.commit(connection) + cls.begin(profile, connection.get('name')) + + to_return = super(PostgresAdapter, cls).drop( + profile, relation, relation_type, model_name) + + cls.commit(connection) + cls.begin(profile, connection.get('name')) + + return to_return + + finally: + drop_lock.release() From 2952eead8e2529b05966e80b5fe88b548bd00133 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sat, 25 Mar 2017 16:11:00 -0400 Subject: [PATCH 2/3] add test proving this works --- .../021_concurrency_test/models/table_a.sql | 7 ++ .../021_concurrency_test/models/table_b.sql | 7 ++ .../models/view_with_conflicting_cascade.sql | 5 + .../021_concurrency_test/test_concurrency.py | 24 +++++ .../021_concurrency_test/update.sql | 101 ++++++++++++++++++ 5 files changed, 144 insertions(+) create mode 100644 test/integration/021_concurrency_test/models/table_a.sql create mode 100644 test/integration/021_concurrency_test/models/table_b.sql create mode 100644 test/integration/021_concurrency_test/models/view_with_conflicting_cascade.sql create mode 100644 test/integration/021_concurrency_test/update.sql diff --git a/test/integration/021_concurrency_test/models/table_a.sql b/test/integration/021_concurrency_test/models/table_a.sql new file mode 100644 index 00000000000..723fdf66582 --- /dev/null +++ b/test/integration/021_concurrency_test/models/table_a.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized = "table" + ) +}} + +select * from "concurrency_021"."seed" diff --git a/test/integration/021_concurrency_test/models/table_b.sql b/test/integration/021_concurrency_test/models/table_b.sql new file mode 100644 index 00000000000..723fdf66582 --- /dev/null +++ b/test/integration/021_concurrency_test/models/table_b.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized = "table" + ) +}} + +select * from "concurrency_021"."seed" diff --git a/test/integration/021_concurrency_test/models/view_with_conflicting_cascade.sql b/test/integration/021_concurrency_test/models/view_with_conflicting_cascade.sql new file mode 100644 index 00000000000..a200f940884 --- /dev/null +++ b/test/integration/021_concurrency_test/models/view_with_conflicting_cascade.sql @@ -0,0 +1,5 @@ +select * from {{ref('table_a')}} + +union all + +select * from {{ref('table_b')}} diff --git a/test/integration/021_concurrency_test/test_concurrency.py b/test/integration/021_concurrency_test/test_concurrency.py index 5e76f1d22c3..b4fd73fec93 100644 --- a/test/integration/021_concurrency_test/test_concurrency.py +++ b/test/integration/021_concurrency_test/test_concurrency.py @@ -25,6 +25,19 @@ def test__postgres__concurrency(self): self.assertTablesEqual("seed", "view") self.assertTablesEqual("seed", "dep") + self.assertTablesEqual("seed", "table_a") + self.assertTablesEqual("seed", "table_b") + self.assertTableDoesNotExist("invalid") + self.assertTableDoesNotExist("skip") + + self.run_sql_file("test/integration/021_concurrency_test/update.sql") + + self.run_dbt() + + self.assertTablesEqual("seed", "view") + self.assertTablesEqual("seed", "dep") + self.assertTablesEqual("seed", "table_a") + self.assertTablesEqual("seed", "table_b") self.assertTableDoesNotExist("invalid") self.assertTableDoesNotExist("skip") @@ -38,3 +51,14 @@ def test__snowflake__concurrency(self): self.assertTablesEqual("seed", "view") self.assertTablesEqual("seed", "dep") + self.assertTablesEqual("seed", "table_a") + self.assertTablesEqual("seed", "table_b") + + self.run_sql_file("test/integration/021_concurrency_test/update.sql") + + self.run_dbt() + + self.assertTablesEqual("seed", "view") + self.assertTablesEqual("seed", "dep") + self.assertTablesEqual("seed", "table_a") + self.assertTablesEqual("seed", "table_b") diff --git a/test/integration/021_concurrency_test/update.sql b/test/integration/021_concurrency_test/update.sql new file mode 100644 index 00000000000..1e5e2b5b462 --- /dev/null +++ b/test/integration/021_concurrency_test/update.sql @@ -0,0 +1,101 @@ +insert into "concurrency_021"."seed" (first_name, last_name, email, gender, ip_address) values +('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'), +('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'), +('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'), +('Jimmy', 'Cooper', 'jcooper3@cargocollective.com', 'Male', '198.24.63.114'), +('Katherine', 'Rice', 'krice4@typepad.com', 'Female', '36.97.186.238'), +('Sarah', 'Ryan', 'sryan5@gnu.org', 'Female', '119.117.152.40'), +('Martin', 'Mcdonald', 'mmcdonald6@opera.com', 'Male', '8.76.38.115'), +('Frank', 'Robinson', 'frobinson7@wunderground.com', 'Male', '186.14.64.194'), +('Jennifer', 'Franklin', 'jfranklin8@mail.ru', 'Female', '91.216.3.131'), +('Henry', 'Welch', 'hwelch9@list-manage.com', 'Male', '176.35.182.168'), +('Fred', 'Snyder', 'fsnydera@reddit.com', 'Male', '217.106.196.54'), +('Amy', 'Dunn', 'adunnb@nba.com', 'Female', '95.39.163.195'), +('Kathleen', 'Meyer', 'kmeyerc@cdc.gov', 'Female', '164.142.188.214'), +('Steve', 'Ferguson', 'sfergusond@reverbnation.com', 'Male', '138.22.204.251'), +('Teresa', 'Hill', 'thille@dion.ne.jp', 'Female', '82.84.228.235'), +('Amanda', 'Harper', 'aharperf@mail.ru', 'Female', '16.123.56.176'), +('Kimberly', 'Ray', 'krayg@xing.com', 'Female', '48.66.48.12'), +('Johnny', 'Knight', 'jknighth@jalbum.net', 'Male', '99.30.138.123'), +('Virginia', 'Freeman', 'vfreemani@tiny.cc', 'Female', '225.172.182.63'), +('Anna', 'Austin', 'aaustinj@diigo.com', 'Female', '62.111.227.148'), +('Willie', 'Hill', 'whillk@mail.ru', 'Male', '0.86.232.249'), +('Sean', 'Harris', 'sharrisl@zdnet.com', 'Male', '117.165.133.249'), +('Mildred', 'Adams', 'madamsm@usatoday.com', 'Female', '163.44.97.46'), +('David', 'Graham', 'dgrahamn@zimbio.com', 'Male', '78.13.246.202'), +('Victor', 'Hunter', 'vhuntero@ehow.com', 'Male', '64.156.179.139'), +('Aaron', 'Ruiz', 'aruizp@weebly.com', 'Male', '34.194.68.78'), +('Benjamin', 'Brooks', 'bbrooksq@jalbum.net', 'Male', '20.192.189.107'), +('Lisa', 'Wilson', 'lwilsonr@japanpost.jp', 'Female', '199.152.130.217'), +('Benjamin', 'King', 'bkings@comsenz.com', 'Male', '29.189.189.213'), +('Christina', 'Williamson', 'cwilliamsont@boston.com', 'Female', '194.101.52.60'), +('Jane', 'Gonzalez', 'jgonzalezu@networksolutions.com', 'Female', '109.119.12.87'), +('Thomas', 'Owens', 'towensv@psu.edu', 'Male', '84.168.213.153'), +('Katherine', 'Moore', 'kmoorew@naver.com', 'Female', '183.150.65.24'), +('Jennifer', 'Stewart', 'jstewartx@yahoo.com', 'Female', '38.41.244.58'), +('Sara', 'Tucker', 'stuckery@topsy.com', 'Female', '181.130.59.184'), +('Harold', 'Ortiz', 'hortizz@vkontakte.ru', 'Male', '198.231.63.137'), +('Shirley', 'James', 'sjames10@yelp.com', 'Female', '83.27.160.104'), +('Dennis', 'Johnson', 'djohnson11@slate.com', 'Male', '183.178.246.101'), +('Louise', 'Weaver', 'lweaver12@china.com.cn', 'Female', '1.14.110.18'), +('Maria', 'Armstrong', 'marmstrong13@prweb.com', 'Female', '181.142.1.249'), +('Gloria', 'Cruz', 'gcruz14@odnoklassniki.ru', 'Female', '178.232.140.243'), +('Diana', 'Spencer', 'dspencer15@ifeng.com', 'Female', '125.153.138.244'), +('Kelly', 'Nguyen', 'knguyen16@altervista.org', 'Female', '170.13.201.119'), +('Jane', 'Rodriguez', 'jrodriguez17@biblegateway.com', 'Female', '12.102.249.81'), +('Scott', 'Brown', 'sbrown18@geocities.jp', 'Male', '108.174.99.192'), +('Norma', 'Cruz', 'ncruz19@si.edu', 'Female', '201.112.156.197'), +('Marie', 'Peters', 'mpeters1a@mlb.com', 'Female', '231.121.197.144'), +('Lillian', 'Carr', 'lcarr1b@typepad.com', 'Female', '206.179.164.163'), +('Judy', 'Nichols', 'jnichols1c@t-online.de', 'Female', '158.190.209.194'), +('Billy', 'Long', 'blong1d@yahoo.com', 'Male', '175.20.23.160'), +('Howard', 'Reid', 'hreid1e@exblog.jp', 'Male', '118.99.196.20'), +('Laura', 'Ferguson', 'lferguson1f@tuttocitta.it', 'Female', '22.77.87.110'), +('Anne', 'Bailey', 'abailey1g@geocities.com', 'Female', '58.144.159.245'), +('Rose', 'Morgan', 'rmorgan1h@ehow.com', 'Female', '118.127.97.4'), +('Nicholas', 'Reyes', 'nreyes1i@google.ru', 'Male', '50.135.10.252'), +('Joshua', 'Kennedy', 'jkennedy1j@house.gov', 'Male', '154.6.163.209'), +('Paul', 'Watkins', 'pwatkins1k@upenn.edu', 'Male', '177.236.120.87'), +('Kathryn', 'Kelly', 'kkelly1l@businessweek.com', 'Female', '70.28.61.86'), +('Adam', 'Armstrong', 'aarmstrong1m@techcrunch.com', 'Male', '133.235.24.202'), +('Norma', 'Wallace', 'nwallace1n@phoca.cz', 'Female', '241.119.227.128'), +('Timothy', 'Reyes', 'treyes1o@google.cn', 'Male', '86.28.23.26'), +('Elizabeth', 'Patterson', 'epatterson1p@sun.com', 'Female', '139.97.159.149'), +('Edward', 'Gomez', 'egomez1q@google.fr', 'Male', '158.103.108.255'), +('David', 'Cox', 'dcox1r@friendfeed.com', 'Male', '206.80.80.58'), +('Brenda', 'Wood', 'bwood1s@over-blog.com', 'Female', '217.207.44.179'), +('Adam', 'Walker', 'awalker1t@blogs.com', 'Male', '253.211.54.93'), +('Michael', 'Hart', 'mhart1u@wix.com', 'Male', '230.206.200.22'), +('Jesse', 'Ellis', 'jellis1v@google.co.uk', 'Male', '213.254.162.52'), +('Janet', 'Powell', 'jpowell1w@un.org', 'Female', '27.192.194.86'), +('Helen', 'Ford', 'hford1x@creativecommons.org', 'Female', '52.160.102.168'), +('Gerald', 'Carpenter', 'gcarpenter1y@about.me', 'Male', '36.30.194.218'), +('Kathryn', 'Oliver', 'koliver1z@army.mil', 'Female', '202.63.103.69'), +('Alan', 'Berry', 'aberry20@gov.uk', 'Male', '246.157.112.211'), +('Harry', 'Andrews', 'handrews21@ameblo.jp', 'Male', '195.108.0.12'), +('Andrea', 'Hall', 'ahall22@hp.com', 'Female', '149.162.163.28'), +('Barbara', 'Wells', 'bwells23@behance.net', 'Female', '224.70.72.1'), +('Anne', 'Wells', 'awells24@apache.org', 'Female', '180.168.81.153'), +('Harry', 'Harper', 'hharper25@rediff.com', 'Male', '151.87.130.21'), +('Jack', 'Ray', 'jray26@wufoo.com', 'Male', '220.109.38.178'), +('Phillip', 'Hamilton', 'phamilton27@joomla.org', 'Male', '166.40.47.30'), +('Shirley', 'Hunter', 'shunter28@newsvine.com', 'Female', '97.209.140.194'), +('Arthur', 'Daniels', 'adaniels29@reuters.com', 'Male', '5.40.240.86'), +('Virginia', 'Rodriguez', 'vrodriguez2a@walmart.com', 'Female', '96.80.164.184'), +('Christina', 'Ryan', 'cryan2b@hibu.com', 'Female', '56.35.5.52'), +('Theresa', 'Mendoza', 'tmendoza2c@vinaora.com', 'Female', '243.42.0.210'), +('Jason', 'Cole', 'jcole2d@ycombinator.com', 'Male', '198.248.39.129'), +('Phillip', 'Bryant', 'pbryant2e@rediff.com', 'Male', '140.39.116.251'), +('Adam', 'Torres', 'atorres2f@sun.com', 'Male', '101.75.187.135'), +('Margaret', 'Johnston', 'mjohnston2g@ucsd.edu', 'Female', '159.30.69.149'), +('Paul', 'Payne', 'ppayne2h@hhs.gov', 'Male', '199.234.140.220'), +('Todd', 'Willis', 'twillis2i@businessweek.com', 'Male', '191.59.136.214'), +('Willie', 'Oliver', 'woliver2j@noaa.gov', 'Male', '44.212.35.197'), +('Frances', 'Robertson', 'frobertson2k@go.com', 'Female', '31.117.65.136'), +('Gregory', 'Hawkins', 'ghawkins2l@joomla.org', 'Male', '91.3.22.49'), +('Lisa', 'Perkins', 'lperkins2m@si.edu', 'Female', '145.95.31.186'), +('Jacqueline', 'Anderson', 'janderson2n@cargocollective.com', 'Female', '14.176.0.187'), +('Shirley', 'Diaz', 'sdiaz2o@ucla.edu', 'Female', '207.12.95.46'), +('Nicole', 'Meyer', 'nmeyer2p@flickr.com', 'Female', '231.79.115.13'), +('Mary', 'Gray', 'mgray2q@constantcontact.com', 'Female', '210.116.64.253'), +('Jean', 'Mcdonald', 'jmcdonald2r@baidu.com', 'Female', '122.239.235.117'); From 0515d4784c723250ac500c31ea2646ff3c27a8f0 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sun, 26 Mar 2017 13:46:40 -0400 Subject: [PATCH 3/3] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3bd1111282..6905ccc37e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Ignore commented-out schema tests ([#330](https://github.com/fishtown-analytics/dbt/pull/330), [#328](https://github.com/fishtown-analytics/dbt/issues/328)) - Fix run levels ([#343](https://github.com/fishtown-analytics/dbt/pull/343), [#340](https://github.com/fishtown-analytics/dbt/issues/340), [#338](https://github.com/fishtown-analytics/dbt/issues/338)) - Fix concurrency, open a unique transaction per model ([#345](https://github.com/fishtown-analytics/dbt/pull/345), [#336](https://github.com/fishtown-analytics/dbt/issues/336)) +- Handle concurrent `DROP ... CASCADE`s in Redshift ([#349](https://github.com/fishtown-analytics/dbt/pull/349)) ### Changes