Handle concurrent DROP ... CASCADEs in Redshift (#349)

dbt-labs · Mar 26, 2017 · e080f73 · e080f73
1 parent d35249c
commit e080f73
Show file tree

Hide file tree

Showing 7 changed files with 175 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@
 - Ignore commented-out schema tests ([#330](https://github.com/fishtown-analytics/dbt/pull/330), [#328](https://github.com/fishtown-analytics/dbt/issues/328))
 - Fix run levels ([#343](https://github.com/fishtown-analytics/dbt/pull/343), [#340](https://github.com/fishtown-analytics/dbt/issues/340), [#338](https://github.com/fishtown-analytics/dbt/issues/338))
 - Fix concurrency, open a unique transaction per model ([#345](https://github.com/fishtown-analytics/dbt/pull/345), [#336](https://github.com/fishtown-analytics/dbt/issues/336))
+- Handle concurrent `DROP ... CASCADE`s in Redshift ([#349](https://github.com/fishtown-analytics/dbt/pull/349))
 
 ### Changes
 

diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift.py
@@ -1,7 +1,12 @@
+import multiprocessing
+
 from dbt.adapters.postgres import PostgresAdapter
 from dbt.logger import GLOBAL_LOGGER as logger  # noqa
 
 
+drop_lock = multiprocessing.Lock()
+
+
 class RedshiftAdapter(PostgresAdapter):
 
     @classmethod
@@ -42,3 +47,28 @@ def sort_qualifier(cls, sort_type, sort):
         return "{sort_type} sortkey({keys_csv})".format(
             sort_type=sort_type, keys_csv=keys_csv
         )
+
+    @classmethod
+    def drop(cls, profile, relation, relation_type, model_name=None):
+        global drop_lock
+
+        to_return = None
+
+        try:
+            drop_lock.acquire()
+
+            connection = cls.get_connection(profile, model_name)
+
+            cls.commit(connection)
+            cls.begin(profile, connection.get('name'))
+
+            to_return = super(PostgresAdapter, cls).drop(
+                profile, relation, relation_type, model_name)
+
+            cls.commit(connection)
+            cls.begin(profile, connection.get('name'))
+
+            return to_return
+
+        finally:
+            drop_lock.release()
diff --git a/test/integration/021_concurrency_test/models/table_a.sql b/test/integration/021_concurrency_test/models/table_a.sql
@@ -0,0 +1,7 @@
+{{
+  config(
+    materialized = "table"
+  )
+}}
+
+select * from "concurrency_021"."seed"
diff --git a/test/integration/021_concurrency_test/models/table_b.sql b/test/integration/021_concurrency_test/models/table_b.sql
@@ -0,0 +1,7 @@
+{{
+  config(
+    materialized = "table"
+  )
+}}
+
+select * from "concurrency_021"."seed"
diff --git a/test/integration/021_concurrency_test/models/view_with_conflicting_cascade.sql b/test/integration/021_concurrency_test/models/view_with_conflicting_cascade.sql
@@ -0,0 +1,5 @@
+select * from {{ref('table_a')}}
+
+union all
+
+select * from {{ref('table_b')}}
diff --git a/test/integration/021_concurrency_test/test_concurrency.py b/test/integration/021_concurrency_test/test_concurrency.py
@@ -25,6 +25,19 @@ def test__postgres__concurrency(self):
 
         self.assertTablesEqual("seed", "view")
         self.assertTablesEqual("seed", "dep")
+        self.assertTablesEqual("seed", "table_a")
+        self.assertTablesEqual("seed", "table_b")
+        self.assertTableDoesNotExist("invalid")
+        self.assertTableDoesNotExist("skip")
+
+        self.run_sql_file("test/integration/021_concurrency_test/update.sql")
+
+        self.run_dbt()
+
+        self.assertTablesEqual("seed", "view")
+        self.assertTablesEqual("seed", "dep")
+        self.assertTablesEqual("seed", "table_a")
+        self.assertTablesEqual("seed", "table_b")
         self.assertTableDoesNotExist("invalid")
         self.assertTableDoesNotExist("skip")
 
@@ -38,3 +51,14 @@ def test__snowflake__concurrency(self):
 
         self.assertTablesEqual("seed", "view")
         self.assertTablesEqual("seed", "dep")
+        self.assertTablesEqual("seed", "table_a")
+        self.assertTablesEqual("seed", "table_b")
+
+        self.run_sql_file("test/integration/021_concurrency_test/update.sql")
+
+        self.run_dbt()
+
+        self.assertTablesEqual("seed", "view")
+        self.assertTablesEqual("seed", "dep")
+        self.assertTablesEqual("seed", "table_a")
+        self.assertTablesEqual("seed", "table_b")
diff --git a/test/integration/021_concurrency_test/update.sql b/test/integration/021_concurrency_test/update.sql
@@ -0,0 +1,101 @@
+insert into "concurrency_021"."seed" (first_name, last_name, email, gender, ip_address) values
+('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'),
+('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'),
+('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'),
+('Jimmy', 'Cooper', 'jcooper3@cargocollective.com', 'Male', '198.24.63.114'),
+('Katherine', 'Rice', 'krice4@typepad.com', 'Female', '36.97.186.238'),
+('Sarah', 'Ryan', 'sryan5@gnu.org', 'Female', '119.117.152.40'),
+('Martin', 'Mcdonald', 'mmcdonald6@opera.com', 'Male', '8.76.38.115'),
+('Frank', 'Robinson', 'frobinson7@wunderground.com', 'Male', '186.14.64.194'),
+('Jennifer', 'Franklin', 'jfranklin8@mail.ru', 'Female', '91.216.3.131'),
+('Henry', 'Welch', 'hwelch9@list-manage.com', 'Male', '176.35.182.168'),
+('Fred', 'Snyder', 'fsnydera@reddit.com', 'Male', '217.106.196.54'),
+('Amy', 'Dunn', 'adunnb@nba.com', 'Female', '95.39.163.195'),
+('Kathleen', 'Meyer', 'kmeyerc@cdc.gov', 'Female', '164.142.188.214'),
+('Steve', 'Ferguson', 'sfergusond@reverbnation.com', 'Male', '138.22.204.251'),
+('Teresa', 'Hill', 'thille@dion.ne.jp', 'Female', '82.84.228.235'),
+('Amanda', 'Harper', 'aharperf@mail.ru', 'Female', '16.123.56.176'),
+('Kimberly', 'Ray', 'krayg@xing.com', 'Female', '48.66.48.12'),
+('Johnny', 'Knight', 'jknighth@jalbum.net', 'Male', '99.30.138.123'),
+('Virginia', 'Freeman', 'vfreemani@tiny.cc', 'Female', '225.172.182.63'),
+('Anna', 'Austin', 'aaustinj@diigo.com', 'Female', '62.111.227.148'),
+('Willie', 'Hill', 'whillk@mail.ru', 'Male', '0.86.232.249'),
+('Sean', 'Harris', 'sharrisl@zdnet.com', 'Male', '117.165.133.249'),
+('Mildred', 'Adams', 'madamsm@usatoday.com', 'Female', '163.44.97.46'),
+('David', 'Graham', 'dgrahamn@zimbio.com', 'Male', '78.13.246.202'),
+('Victor', 'Hunter', 'vhuntero@ehow.com', 'Male', '64.156.179.139'),
+('Aaron', 'Ruiz', 'aruizp@weebly.com', 'Male', '34.194.68.78'),
+('Benjamin', 'Brooks', 'bbrooksq@jalbum.net', 'Male', '20.192.189.107'),
+('Lisa', 'Wilson', 'lwilsonr@japanpost.jp', 'Female', '199.152.130.217'),
+('Benjamin', 'King', 'bkings@comsenz.com', 'Male', '29.189.189.213'),
+('Christina', 'Williamson', 'cwilliamsont@boston.com', 'Female', '194.101.52.60'),
+('Jane', 'Gonzalez', 'jgonzalezu@networksolutions.com', 'Female', '109.119.12.87'),
+('Thomas', 'Owens', 'towensv@psu.edu', 'Male', '84.168.213.153'),
+('Katherine', 'Moore', 'kmoorew@naver.com', 'Female', '183.150.65.24'),
+('Jennifer', 'Stewart', 'jstewartx@yahoo.com', 'Female', '38.41.244.58'),
+('Sara', 'Tucker', 'stuckery@topsy.com', 'Female', '181.130.59.184'),
+('Harold', 'Ortiz', 'hortizz@vkontakte.ru', 'Male', '198.231.63.137'),
+('Shirley', 'James', 'sjames10@yelp.com', 'Female', '83.27.160.104'),
+('Dennis', 'Johnson', 'djohnson11@slate.com', 'Male', '183.178.246.101'),
+('Louise', 'Weaver', 'lweaver12@china.com.cn', 'Female', '1.14.110.18'),
+('Maria', 'Armstrong', 'marmstrong13@prweb.com', 'Female', '181.142.1.249'),
+('Gloria', 'Cruz', 'gcruz14@odnoklassniki.ru', 'Female', '178.232.140.243'),
+('Diana', 'Spencer', 'dspencer15@ifeng.com', 'Female', '125.153.138.244'),
+('Kelly', 'Nguyen', 'knguyen16@altervista.org', 'Female', '170.13.201.119'),
+('Jane', 'Rodriguez', 'jrodriguez17@biblegateway.com', 'Female', '12.102.249.81'),
+('Scott', 'Brown', 'sbrown18@geocities.jp', 'Male', '108.174.99.192'),
+('Norma', 'Cruz', 'ncruz19@si.edu', 'Female', '201.112.156.197'),
+('Marie', 'Peters', 'mpeters1a@mlb.com', 'Female', '231.121.197.144'),
+('Lillian', 'Carr', 'lcarr1b@typepad.com', 'Female', '206.179.164.163'),
+('Judy', 'Nichols', 'jnichols1c@t-online.de', 'Female', '158.190.209.194'),
+('Billy', 'Long', 'blong1d@yahoo.com', 'Male', '175.20.23.160'),
+('Howard', 'Reid', 'hreid1e@exblog.jp', 'Male', '118.99.196.20'),
+('Laura', 'Ferguson', 'lferguson1f@tuttocitta.it', 'Female', '22.77.87.110'),
+('Anne', 'Bailey', 'abailey1g@geocities.com', 'Female', '58.144.159.245'),
+('Rose', 'Morgan', 'rmorgan1h@ehow.com', 'Female', '118.127.97.4'),
+('Nicholas', 'Reyes', 'nreyes1i@google.ru', 'Male', '50.135.10.252'),
+('Joshua', 'Kennedy', 'jkennedy1j@house.gov', 'Male', '154.6.163.209'),
+('Paul', 'Watkins', 'pwatkins1k@upenn.edu', 'Male', '177.236.120.87'),
+('Kathryn', 'Kelly', 'kkelly1l@businessweek.com', 'Female', '70.28.61.86'),
+('Adam', 'Armstrong', 'aarmstrong1m@techcrunch.com', 'Male', '133.235.24.202'),
+('Norma', 'Wallace', 'nwallace1n@phoca.cz', 'Female', '241.119.227.128'),
+('Timothy', 'Reyes', 'treyes1o@google.cn', 'Male', '86.28.23.26'),
+('Elizabeth', 'Patterson', 'epatterson1p@sun.com', 'Female', '139.97.159.149'),
+('Edward', 'Gomez', 'egomez1q@google.fr', 'Male', '158.103.108.255'),
+('David', 'Cox', 'dcox1r@friendfeed.com', 'Male', '206.80.80.58'),
+('Brenda', 'Wood', 'bwood1s@over-blog.com', 'Female', '217.207.44.179'),
+('Adam', 'Walker', 'awalker1t@blogs.com', 'Male', '253.211.54.93'),
+('Michael', 'Hart', 'mhart1u@wix.com', 'Male', '230.206.200.22'),
+('Jesse', 'Ellis', 'jellis1v@google.co.uk', 'Male', '213.254.162.52'),
+('Janet', 'Powell', 'jpowell1w@un.org', 'Female', '27.192.194.86'),
+('Helen', 'Ford', 'hford1x@creativecommons.org', 'Female', '52.160.102.168'),
+('Gerald', 'Carpenter', 'gcarpenter1y@about.me', 'Male', '36.30.194.218'),
+('Kathryn', 'Oliver', 'koliver1z@army.mil', 'Female', '202.63.103.69'),
+('Alan', 'Berry', 'aberry20@gov.uk', 'Male', '246.157.112.211'),
+('Harry', 'Andrews', 'handrews21@ameblo.jp', 'Male', '195.108.0.12'),
+('Andrea', 'Hall', 'ahall22@hp.com', 'Female', '149.162.163.28'),
+('Barbara', 'Wells', 'bwells23@behance.net', 'Female', '224.70.72.1'),
+('Anne', 'Wells', 'awells24@apache.org', 'Female', '180.168.81.153'),
+('Harry', 'Harper', 'hharper25@rediff.com', 'Male', '151.87.130.21'),
+('Jack', 'Ray', 'jray26@wufoo.com', 'Male', '220.109.38.178'),
+('Phillip', 'Hamilton', 'phamilton27@joomla.org', 'Male', '166.40.47.30'),
+('Shirley', 'Hunter', 'shunter28@newsvine.com', 'Female', '97.209.140.194'),
+('Arthur', 'Daniels', 'adaniels29@reuters.com', 'Male', '5.40.240.86'),
+('Virginia', 'Rodriguez', 'vrodriguez2a@walmart.com', 'Female', '96.80.164.184'),
+('Christina', 'Ryan', 'cryan2b@hibu.com', 'Female', '56.35.5.52'),
+('Theresa', 'Mendoza', 'tmendoza2c@vinaora.com', 'Female', '243.42.0.210'),
+('Jason', 'Cole', 'jcole2d@ycombinator.com', 'Male', '198.248.39.129'),
+('Phillip', 'Bryant', 'pbryant2e@rediff.com', 'Male', '140.39.116.251'),
+('Adam', 'Torres', 'atorres2f@sun.com', 'Male', '101.75.187.135'),
+('Margaret', 'Johnston', 'mjohnston2g@ucsd.edu', 'Female', '159.30.69.149'),
+('Paul', 'Payne', 'ppayne2h@hhs.gov', 'Male', '199.234.140.220'),
+('Todd', 'Willis', 'twillis2i@businessweek.com', 'Male', '191.59.136.214'),
+('Willie', 'Oliver', 'woliver2j@noaa.gov', 'Male', '44.212.35.197'),
+('Frances', 'Robertson', 'frobertson2k@go.com', 'Female', '31.117.65.136'),
+('Gregory', 'Hawkins', 'ghawkins2l@joomla.org', 'Male', '91.3.22.49'),
+('Lisa', 'Perkins', 'lperkins2m@si.edu', 'Female', '145.95.31.186'),
+('Jacqueline', 'Anderson', 'janderson2n@cargocollective.com', 'Female', '14.176.0.187'),
+('Shirley', 'Diaz', 'sdiaz2o@ucla.edu', 'Female', '207.12.95.46'),
+('Nicole', 'Meyer', 'nmeyer2p@flickr.com', 'Female', '231.79.115.13'),
+('Mary', 'Gray', 'mgray2q@constantcontact.com', 'Female', '210.116.64.253'),
+('Jean', 'Mcdonald', 'jmcdonald2r@baidu.com', 'Female', '122.239.235.117');