Merge pull request #2280 from fishtown-analytics/fix/add-partitions-b…

…q-config

Add "partitions" to the adapter specific config (#2256)

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Fix an issue with raw blocks where multiple raw blocks in the same file resulted in an error ([#2241](https://github.com/fishtown-analytics/dbt/issues/2241), [#2252](https://github.com/fishtown-analytics/dbt/pull/2252))
 - Fix a redshift-only issue that caused an error when `dbt seed` found a seed with an entirely empty column that was set to a `varchar` data type. ([#2250](https://github.com/fishtown-analytics/dbt/issues/2250), [#2254](https://github.com/fishtown-analytics/dbt/pull/2254))
 - Fix a bug where third party plugins that used the default `list_schemas` and `information_schema_name` macros with database quoting enabled double-quoted the database name in their queries ([#2267](https://github.com/fishtown-analytics/dbt/issues/2267), [#2281](https://github.com/fishtown-analytics/dbt/pull/2281))
+- The BigQuery "partitions" config value can now be used in `dbt_project.yml` ([##2256](https://github.com/fishtown-analytics/dbt/issues/#2256), [#2280](https://github.com/fishtown-analytics/dbt/pull/2280))
 
 ### Under the hood
 - Pin google libraries to higher minimum values, add more dependencies as explicit ([#2233](https://github.com/fishtown-analytics/dbt/issues/2233), [#2249](https://github.com/fishtown-analytics/dbt/pull/2249))

plugins/bigquery/dbt/adapters/bigquery/impl.py

@@ -138,8 +138,9 @@ class BigQueryAdapter(BaseAdapter):
     Column = BigQueryColumn
     ConnectionManager = BigQueryConnectionManager
 
-    AdapterSpecificConfigs = frozenset({"cluster_by", "partition_by",
-                                        "kms_key_name", "labels"})
+    AdapterSpecificConfigs = frozenset({
+        "cluster_by", "partition_by", "kms_key_name", "labels", "partitions"
+    })
 
     ###
     # Implementations of abstract methods

test/integration/022_bigquery_test/dp-models/confirmation_noconfig.sql

@@ -0,0 +1,20 @@
+
+-- This model checks to confirm that each date partition was created correctly.
+-- Columns day_1, day_2, and day_3 should have a value of 1, and count_days should be 3
+
+with base as (
+
+  select
+    case when _PARTITIONTIME = '2018-01-01' then 1 else 0 end as day_1,
+    case when _PARTITIONTIME = '2018-01-02' then 1 else 0 end as day_2,
+    case when _PARTITIONTIME = '2018-01-03' then 1 else 0 end as day_3
+  from {{ ref('partitioned_noconfig') }}
+
+)
+
+select distinct
+  sum(day_1) over () as day_1,
+  sum(day_2) over () as day_2,
+  sum(day_3) over () as day_3,
+  count(*) over () as count_days
+from base

test/integration/022_bigquery_test/dp-models/partitioned_noconfig.sql

@@ -0,0 +1,7 @@
+-- Hack to make sure our events models run first.
+-- In practice, these would be source data
+-- {{ ref('events_20180101') }}
+-- {{ ref('events_20180102') }}
+-- {{ ref('events_20180103') }}
+
+select * from `{{ this.schema }}`.`{{ date_sharded_table('events_') }}`

test/integration/022_bigquery_test/dp-models/schema.yml

@@ -7,7 +7,7 @@ models:
     - not_null
     - unique
 - name: confirmation
-  columns:
+  columns: &conf_columns
   - name: cast(count_days as string)
     tests:
     - accepted_values:
@@ -28,3 +28,6 @@ models:
     - accepted_values:
         values:
         - 1
+
+- name: confirmation_noconfig
+  columns: *conf_columns

test/integration/022_bigquery_test/test_bigquery_date_partitioning.py

@@ -1,4 +1,6 @@
 from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile
+import textwrap
+import yaml
 
 
 class TestBigqueryDatePartitioning(DBTIntegrationTest):
@@ -15,10 +17,24 @@ def models(self):
     def profile_config(self):
         return self.bigquery_profile()
 
+    @property
+    def project_config(self):
+        return yaml.safe_load(textwrap.dedent('''\
+        models:
+            test:
+                partitioned_noconfig:
+                    materialized: table
+                    partitions:
+                        - 20180101
+                        - 20180102
+                        - 20180103
+                    verbose: true
+        '''))
+
     @use_profile('bigquery')
     def test__bigquery_date_partitioning(self):
         results = self.run_dbt()
-        self.assertEqual(len(results), 6)
+        self.assertEqual(len(results), 8)
 
         test_results = self.run_dbt(['test'])