From 3829d1f964c25bb852565fae45dac593b09659dd Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 17 Jun 2020 07:39:41 -0600 Subject: [PATCH 1/4] Fix a regression where dbt ignored aliases in config() calls --- CHANGELOG.md | 1 + core/dbt/context/context_config.py | 3 ++- .../040_override_database_test/models/view_2.sql | 8 +++++--- test/integration/base.py | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba8aeff50b2..cfc2389c426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - `packages.yml` revision/version fields can be float-like again (`revision: '1.0'` is valid). ([#2518](https://github.com/fishtown-analytics/dbt/issues/2518), [#2535](https://github.com/fishtown-analytics/dbt/pull/2535)) - Parallel RPC requests no longer step on each others' arguments ([[#2484](https://github.com/fishtown-analytics/dbt/issues/2484), [#2554](https://github.com/fishtown-analytics/dbt/pull/2554)]) - On windows (depending upon OS support), dbt no longer fails with errors when writing artifacts ([#2558](https://github.com/fishtown-analytics/dbt/issues/2558), [#2566](https://github.com/fishtown-analytics/dbt/pull/2566)) +- dbt again respects config aliases in config() calls ([#2557](https://github.com/fishtown-analytics/dbt/issues/2557), [#2559](https://github.com/fishtown-analytics/dbt/pull/2559)) ## dbt 0.17.0 (June 08, 2020) diff --git a/core/dbt/context/context_config.py b/core/dbt/context/context_config.py index a33be3357e4..3cdf9bf92e1 100644 --- a/core/dbt/context/context_config.py +++ b/core/dbt/context/context_config.py @@ -131,8 +131,9 @@ def active_project_configs( def _update_from_config( self, result: T, partial: Dict[str, Any], validate: bool = False ) -> T: + translated = self.active_project.credentials.translate_aliases(partial) return result.update_from( - partial, + translated, self.active_project.credentials.type, validate=validate ) diff --git a/test/integration/040_override_database_test/models/view_2.sql b/test/integration/040_override_database_test/models/view_2.sql index bdfa5369fa7..9ac6bdad6a7 100644 --- a/test/integration/040_override_database_test/models/view_2.sql +++ b/test/integration/040_override_database_test/models/view_2.sql @@ -1,4 +1,6 @@ -{{ - config(database=var('alternate_db')) -}} +{%- if target.type == 'bigquery' -%} + {{ config(project=var('alternate_db')) }} +{%- else -%} + {{ config(database=var('alternate_db')) }} +{%- endif -%} select * from {{ ref('seed') }} diff --git a/test/integration/base.py b/test/integration/base.py index 0ca24bf8b94..6ae5a62fe2e 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -1024,6 +1024,8 @@ def assertManyRelationsEqual(self, relations, default_schema=None, default_datab first_columns = None for relation in specs: key = (relation.database, relation.schema, relation.identifier) + # get a good error here instead of a hard-to-diagnose KeyError + self.assertIn(key, column_specs, f'No columns found for {key}') columns = column_specs[key] if first_columns is None: first_columns = columns From e300c62d763cedb69e78ce6fc0d56c996eb4bbcb Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 19 Jun 2020 11:24:47 -0600 Subject: [PATCH 2/4] add a test --- .../test_override_database.py | 66 +++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/test/integration/040_override_database_test/test_override_database.py b/test/integration/040_override_database_test/test_override_database.py index 37ea59b8eb5..085750a8b15 100644 --- a/test/integration/040_override_database_test/test_override_database.py +++ b/test/integration/040_override_database_test/test_override_database.py @@ -99,13 +99,44 @@ def test_snowflake_database_override(self): self.run_database_override() -class TestProjectModelOverride(BaseOverrideDatabase): +class BaseTestProjectModelOverride(BaseOverrideDatabase): + # this is janky, but I really want to access self.default_database in + # project_config + @property + def default_database(self): + target = self._profile_config['test']['target'] + profile = self._profile_config['test']['outputs'][target] + for key in ['database', 'project', 'dbname']: + if key in profile: + database = profile[key] + if self.adapter_type == 'snowflake': + return database.upper() + return database + assert False, 'No profile database found!' + def run_database_override(self): + self.run_dbt_notstrict(['seed']) + self.assertEqual(len(self.run_dbt_notstrict(['run'])), 4) + self.assertExpectedRelations() + + def assertExpectedRelations(self): if self.adapter_type == 'snowflake': func = lambda x: x.upper() else: func = lambda x: x + self.assertManyRelationsEqual([ + (func('seed'), self.unique_schema(), self.default_database), + (func('view_2'), self.unique_schema(), self.alternative_database), + (func('view_1'), self.unique_schema(), self.alternative_database), + (func('view_3'), self.unique_schema(), self.default_database), + (func('view_4'), self.unique_schema(), self.alternative_database), + ]) + + +class TestProjectModelOverride(BaseTestProjectModelOverride): + @property + def project_config(self): self.use_default_project({ 'config-version': 2, 'vars': { @@ -120,16 +151,6 @@ def run_database_override(self): } }, }) - self.run_dbt_notstrict(['seed']) - - self.assertEqual(len(self.run_dbt_notstrict(['run'])), 4) - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.default_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.alternative_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) @use_profile('bigquery') def test_bigquery_database_override(self): @@ -140,6 +161,29 @@ def test_snowflake_database_override(self): self.run_database_override() +class TestProjectModelAliasOverride(BaseTestProjectModelOverride): + @property + def project_config(self): + return { + 'config-version': 2, + 'vars': { + 'alternate_db': self.alternative_database, + }, + 'models': { + 'project': self.alternative_database, + 'test': { + 'subfolder': { + 'project': self.default_database, + } + } + }, + } + + @use_profile('bigquery') + def test_bigquery_project_override(self): + self.run_database_override() + + class TestProjectSeedOverride(BaseOverrideDatabase): def run_database_override(self): if self.adapter_type == 'snowflake': From 05917c6d2ffcbca346dfa9e45f9c7c81c7ebf5f1 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 19 Jun 2020 11:29:35 -0600 Subject: [PATCH 3/4] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfc2389c426..712835010c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - `packages.yml` revision/version fields can be float-like again (`revision: '1.0'` is valid). ([#2518](https://github.com/fishtown-analytics/dbt/issues/2518), [#2535](https://github.com/fishtown-analytics/dbt/pull/2535)) - Parallel RPC requests no longer step on each others' arguments ([[#2484](https://github.com/fishtown-analytics/dbt/issues/2484), [#2554](https://github.com/fishtown-analytics/dbt/pull/2554)]) - On windows (depending upon OS support), dbt no longer fails with errors when writing artifacts ([#2558](https://github.com/fishtown-analytics/dbt/issues/2558), [#2566](https://github.com/fishtown-analytics/dbt/pull/2566)) -- dbt again respects config aliases in config() calls ([#2557](https://github.com/fishtown-analytics/dbt/issues/2557), [#2559](https://github.com/fishtown-analytics/dbt/pull/2559)) +- dbt again respects config aliases in config() calls and dbt_project.yml ([#2557](https://github.com/fishtown-analytics/dbt/issues/2557), [#2559](https://github.com/fishtown-analytics/dbt/pull/2559), [#2575](https://github.com/fishtown-analytics/dbt/pull/2575)) ## dbt 0.17.0 (June 08, 2020) From 098d05c27fd157595ab8f511c7eae6085713f7ad Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 19 Jun 2020 11:52:43 -0600 Subject: [PATCH 4/4] Fix tests --- .../test_override_database.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/integration/040_override_database_test/test_override_database.py b/test/integration/040_override_database_test/test_override_database.py index 085750a8b15..1686ca1214e 100644 --- a/test/integration/040_override_database_test/test_override_database.py +++ b/test/integration/040_override_database_test/test_override_database.py @@ -137,7 +137,7 @@ def assertExpectedRelations(self): class TestProjectModelOverride(BaseTestProjectModelOverride): @property def project_config(self): - self.use_default_project({ + return { 'config-version': 2, 'vars': { 'alternate_db': self.alternative_database, @@ -150,7 +150,17 @@ def project_config(self): } } }, - }) + 'data-paths': ['data'], + 'vars': { + 'alternate_db': self.alternative_database, + }, + 'quoting': { + 'database': True, + }, + 'seeds': { + 'quote_columns': False, + } + } @use_profile('bigquery') def test_bigquery_database_override(self): @@ -177,6 +187,16 @@ def project_config(self): } } }, + 'data-paths': ['data'], + 'vars': { + 'alternate_db': self.alternative_database, + }, + 'quoting': { + 'database': True, + }, + 'seeds': { + 'quote_columns': False, + } } @use_profile('bigquery')