From 8782714ec3e0dadd86d6e439615de4f0d0168c94 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Thu, 25 Jun 2020 10:17:24 -0400 Subject: [PATCH 1/8] Added support for setting policy tags for columns in BigQuery --- CHANGELOG.md | 3 +- docker/requirements/requirements.0.18.0b1.txt | 2 +- .../bigquery/dbt/adapters/bigquery/impl.py | 16 +++- .../dbt/include/bigquery/macros/adapters.sql | 2 +- plugins/bigquery/setup.py | 2 +- test.env.sample | 1 + .../test_bigquery_update_columns.py | 86 +++++++++++++++++++ .../description_table.sql | 9 ++ .../description_table.yml | 7 ++ .../policy_tag_table.sql | 9 ++ .../policy_tag_table.yml | 8 ++ 11 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 test/integration/022_bigquery_test/test_bigquery_update_columns.py create mode 100644 test/integration/022_bigquery_test/update-column-description/description_table.sql create mode 100644 test/integration/022_bigquery_test/update-column-description/description_table.yml create mode 100644 test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.sql create mode 100644 test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3261d4c6c17..8b9637ad44e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ Contributors: - Added support for BigQuery authorized views ([#1718](https://github.com/fishtown-analytics/dbt/issues/1718), [#2517](https://github.com/fishtown-analytics/dbt/pull/2517)) - Added support for altering BigQuery column types ([#2546](https://github.com/fishtown-analytics/dbt/issues/2546), [#2547](https://github.com/fishtown-analytics/dbt/pull/2547)) - Include row counts and bytes processed in log output for all BigQuery statement types ([#2526](https://github.com/fishtown-analytics/dbt/issues/2526)) +- Added support for setting policy tags for columns in BigQuery ([#2586](https://github.com/fishtown-analytics/dbt/issues/2586), [#2588](https://github.com/fishtown-analytics/dbt/pull/2588)) ### Fixes @@ -44,7 +45,7 @@ Contributors: - [@alf-mindshift](https://github.com/alf-mindshift) ([#2431](https://github.com/fishtown-analytics/dbt/pull/2431)) - [@scarrucciu](https://github.com/scarrucciu) ([#2508](https://github.com/fishtown-analytics/dbt/pull/2508)) - [@southpolemonkey](https://github.com/southpolemonkey) ([#2511](https://github.com/fishtown-analytics/dbt/issues/2511)) - - [@azhard](https://github.com/azhard) ([#2517](https://github.com/fishtown-analytics/dbt/pull/2517), ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521)), [#2547](https://github.com/fishtown-analytics/dbt/pull/2547)) + - [@azhard](https://github.com/azhard) ([#2517](https://github.com/fishtown-analytics/dbt/pull/2517), ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521)), [#2547](https://github.com/fishtown-analytics/dbt/pull/2547), [#2588](https://github.com/fishtown-analytics/dbt/pull/2588)) - [@alepuccetti](https://github.com/alepuccetti) ([#2526](https://github.com/fishtown-analytics/dbt/issues/2526)) diff --git a/docker/requirements/requirements.0.18.0b1.txt b/docker/requirements/requirements.0.18.0b1.txt index f0a37f41326..12cc63c7a4b 100644 --- a/docker/requirements/requirements.0.18.0b1.txt +++ b/docker/requirements/requirements.0.18.0b1.txt @@ -17,7 +17,7 @@ decorator==4.4.2 docutils==0.15.2 google-api-core==1.16.0 google-auth==1.16.1 -google-cloud-bigquery==1.24.0 +google-cloud-bigquery==1.25.0 google-cloud-core==1.3.0 google-resumable-media==0.5.1 googleapis-common-protos==1.6.0 diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index 250407e7ccf..f9f413f0be0 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -612,6 +612,10 @@ def _update_column_dict(self, bq_column_dict, dbt_columns, parent=''): if dotted_column_name in dbt_columns: column_config = dbt_columns[dotted_column_name] bq_column_dict['description'] = column_config.get('description') + if column_config.get('tags'): + bq_column_dict['policyTags'] = {} + bq_column_dict['policyTags']['names'] = \ + column_config.get('tags') new_fields = [] for child_col_dict in bq_column_dict.get('fields', list()): @@ -627,7 +631,7 @@ def _update_column_dict(self, bq_column_dict, dbt_columns, parent=''): return bq_column_dict @available.parse_none - def update_column_descriptions(self, relation, columns): + def update_column(self, relation, columns): if len(columns) == 0: return @@ -643,6 +647,16 @@ def update_column_descriptions(self, relation, columns): columns ) new_schema.append(SchemaField.from_api_repr(new_bq_column_dict)) + # for column in table.schema: + # if column.name in columns: + # column_config = columns[column.name] + # column_dict = column.to_api_repr() + # column_dict['description'] = column_config.get('description') + # if column_config.get('tags'): + # column_dict['policyTags'] = {} + # column_dict['policyTags']['names'] = column_config.get('tags') + # column = SchemaField.from_api_repr(column_dict) + # new_schema.append(column) new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema) conn.handle.update_table(new_table, ['schema']) diff --git a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql index 1fda4609cda..37cb0409023 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql @@ -121,7 +121,7 @@ {% endmacro %} {% macro bigquery__alter_column_comment(relation, column_dict) -%} - {% do adapter.update_column_descriptions(relation, column_dict) %} + {% do adapter.update_column(relation, column_dict) %} {% endmacro %} {% macro bigquery__rename_relation(from_relation, to_relation) -%} diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py index 7b827466523..c9cebb7fdba 100644 --- a/plugins/bigquery/setup.py +++ b/plugins/bigquery/setup.py @@ -42,7 +42,7 @@ 'dbt-core=={}'.format(package_version), 'protobuf>=3.6.0,<3.12', 'google-cloud-core>=1.3.0,<1.4', - 'google-cloud-bigquery>=1.24.0,<1.25.0', + 'google-cloud-bigquery>=1.25.0,<1.26.0', 'google-api-core>=1.16.0,<1.17.0', 'googleapis-common-protos>=1.6.0,<1.7.0', 'six>=1.14.0', diff --git a/test.env.sample b/test.env.sample index 2ddf896f8f8..f981f3a4d15 100644 --- a/test.env.sample +++ b/test.env.sample @@ -14,6 +14,7 @@ SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET= BIGQUERY_SERVICE_ACCOUNT_JSON= BIGQUERY_TEST_ALT_DATABASE= +BIGQUERY_POLICY_TAG= REDSHIFT_TEST_HOST= REDSHIFT_TEST_USER= diff --git a/test/integration/022_bigquery_test/test_bigquery_update_columns.py b/test/integration/022_bigquery_test/test_bigquery_update_columns.py new file mode 100644 index 00000000000..48baa39e3a8 --- /dev/null +++ b/test/integration/022_bigquery_test/test_bigquery_update_columns.py @@ -0,0 +1,86 @@ +import os + +from test.integration.base import DBTIntegrationTest, use_profile + + +class TestBigqueryUpdateColumnPolicyTag(DBTIntegrationTest): + + @property + def schema(self): + return "bigquery_test_022" + + @property + def models(self): + return "update-column-policy-tag" + + @property + def project_config(self): + return { + 'config-version': 2, + 'vars': { + 'policy_tag': self.policy_tag + } + } + + @property + def policy_tag(self): + return os.environ['BIGQUERY_POLICY_TAG'] + + @use_profile('bigquery') + def test__bigquery_update_column_policy_tag(self): + if self.policy_tag: + results = self.run_dbt(['run', '--models', 'policy_tag_table']) + self.assertEqual(len(results), 1) + + with self.get_connection() as conn: + client = conn.handle + + table = client.get_table( + self.adapter.connections.get_bq_table( + self.default_database, self.unique_schema(), 'policy_tag_table') + ) + + for schema_field in table.schema: + if self.policy_tag: + self.assertEquals(schema_field.policy_tags.names, + (self.policy_tag,)) + + +class TestBigqueryUpdateColumnDescription(DBTIntegrationTest): + + @property + def schema(self): + return "bigquery_test_022" + + @property + def models(self): + return "update-column-description" + + @property + def project_config(self): + return { + 'config-version': 2, + 'vars': { + 'field_description': self.field_description + } + } + + @property + def field_description(self): + return 'this is a field' + + @use_profile('bigquery') + def test__bigquery_update_column_description(self): + results = self.run_dbt(['run', '--models', 'description_table']) + self.assertEqual(len(results), 1) + + with self.get_connection() as conn: + client = conn.handle + + table = client.get_table( + self.adapter.connections.get_bq_table( + self.default_database, self.unique_schema(), 'description_table') + ) + + for schema_field in table.schema: + self.assertEquals(schema_field.description, self.field_description) diff --git a/test/integration/022_bigquery_test/update-column-description/description_table.sql b/test/integration/022_bigquery_test/update-column-description/description_table.sql new file mode 100644 index 00000000000..7110ac8b0f5 --- /dev/null +++ b/test/integration/022_bigquery_test/update-column-description/description_table.sql @@ -0,0 +1,9 @@ +{{ + config( + materialized='table', + persist_docs={ 'columns': true } + ) +}} + +select + 1 field diff --git a/test/integration/022_bigquery_test/update-column-description/description_table.yml b/test/integration/022_bigquery_test/update-column-description/description_table.yml new file mode 100644 index 00000000000..28d68f0076b --- /dev/null +++ b/test/integration/022_bigquery_test/update-column-description/description_table.yml @@ -0,0 +1,7 @@ +version: 2 + +models: +- name: description_table + columns: + - name: field + description: '{{ var("field_description") }}' diff --git a/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.sql b/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.sql new file mode 100644 index 00000000000..7110ac8b0f5 --- /dev/null +++ b/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.sql @@ -0,0 +1,9 @@ +{{ + config( + materialized='table', + persist_docs={ 'columns': true } + ) +}} + +select + 1 field diff --git a/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml b/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml new file mode 100644 index 00000000000..649fe6f14cf --- /dev/null +++ b/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml @@ -0,0 +1,8 @@ +version: 2 + +models: +- name: policy_tag_table + columns: + - name: field + tags: + - '{{ var("policy_tag") }}' From e3193c6243765b8881cd3898e65d91fab33779b6 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Thu, 25 Jun 2020 10:19:41 -0400 Subject: [PATCH 2/8] Updated pull request # --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b9637ad44e..619246b59c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ Contributors: - Added support for BigQuery authorized views ([#1718](https://github.com/fishtown-analytics/dbt/issues/1718), [#2517](https://github.com/fishtown-analytics/dbt/pull/2517)) - Added support for altering BigQuery column types ([#2546](https://github.com/fishtown-analytics/dbt/issues/2546), [#2547](https://github.com/fishtown-analytics/dbt/pull/2547)) - Include row counts and bytes processed in log output for all BigQuery statement types ([#2526](https://github.com/fishtown-analytics/dbt/issues/2526)) -- Added support for setting policy tags for columns in BigQuery ([#2586](https://github.com/fishtown-analytics/dbt/issues/2586), [#2588](https://github.com/fishtown-analytics/dbt/pull/2588)) +- Added support for setting policy tags for columns in BigQuery ([#2586](https://github.com/fishtown-analytics/dbt/issues/2586), [#2589](https://github.com/fishtown-analytics/dbt/pull/2589)) ### Fixes From 14233188be9dda5a42d65e5e4e0e50a5fbe32bff Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Thu, 25 Jun 2020 10:22:21 -0400 Subject: [PATCH 3/8] Changed update_column to update_columns --- plugins/bigquery/dbt/adapters/bigquery/impl.py | 2 +- plugins/bigquery/dbt/include/bigquery/macros/adapters.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index f9f413f0be0..062fd609962 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -631,7 +631,7 @@ def _update_column_dict(self, bq_column_dict, dbt_columns, parent=''): return bq_column_dict @available.parse_none - def update_column(self, relation, columns): + def update_columns(self, relation, columns): if len(columns) == 0: return diff --git a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql index 37cb0409023..3bb7cd606f2 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql @@ -121,7 +121,7 @@ {% endmacro %} {% macro bigquery__alter_column_comment(relation, column_dict) -%} - {% do adapter.update_column(relation, column_dict) %} + {% do adapter.update_columns(relation, column_dict) %} {% endmacro %} {% macro bigquery__rename_relation(from_relation, to_relation) -%} From b964f6dd2235a60a7b7cc9a0349c6c0d2e08c1cf Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Thu, 25 Jun 2020 10:25:13 -0400 Subject: [PATCH 4/8] Rebased --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 619246b59c9..518d1c0e478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ Contributors: - Added support for BigQuery authorized views ([#1718](https://github.com/fishtown-analytics/dbt/issues/1718), [#2517](https://github.com/fishtown-analytics/dbt/pull/2517)) - Added support for altering BigQuery column types ([#2546](https://github.com/fishtown-analytics/dbt/issues/2546), [#2547](https://github.com/fishtown-analytics/dbt/pull/2547)) - Include row counts and bytes processed in log output for all BigQuery statement types ([#2526](https://github.com/fishtown-analytics/dbt/issues/2526)) -- Added support for setting policy tags for columns in BigQuery ([#2586](https://github.com/fishtown-analytics/dbt/issues/2586), [#2589](https://github.com/fishtown-analytics/dbt/pull/2589)) +- Added support for setting policy tags for BigQuery columns ([#2586](https://github.com/fishtown-analytics/dbt/issues/2586), [#2589](https://github.com/fishtown-analytics/dbt/pull/2589)) ### Fixes From 0a647083026e11b5359416a61db168f558ef3bad Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Thu, 25 Jun 2020 10:36:35 -0400 Subject: [PATCH 5/8] Linting fix --- plugins/bigquery/dbt/adapters/bigquery/impl.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index 062fd609962..67357ad5e28 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -647,16 +647,6 @@ def update_columns(self, relation, columns): columns ) new_schema.append(SchemaField.from_api_repr(new_bq_column_dict)) - # for column in table.schema: - # if column.name in columns: - # column_config = columns[column.name] - # column_dict = column.to_api_repr() - # column_dict['description'] = column_config.get('description') - # if column_config.get('tags'): - # column_dict['policyTags'] = {} - # column_dict['policyTags']['names'] = column_config.get('tags') - # column = SchemaField.from_api_repr(column_dict) - # new_schema.append(column) new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema) conn.handle.update_table(new_table, ['schema']) From 8ec93d99739605dff13f9488e1cc91bca6d77585 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Thu, 25 Jun 2020 10:55:06 -0400 Subject: [PATCH 6/8] bigquery integration test fix --- .../022_bigquery_test/test_bigquery_update_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/022_bigquery_test/test_bigquery_update_columns.py b/test/integration/022_bigquery_test/test_bigquery_update_columns.py index 48baa39e3a8..9ee007666b0 100644 --- a/test/integration/022_bigquery_test/test_bigquery_update_columns.py +++ b/test/integration/022_bigquery_test/test_bigquery_update_columns.py @@ -24,7 +24,7 @@ def project_config(self): @property def policy_tag(self): - return os.environ['BIGQUERY_POLICY_TAG'] + return os.environ.get('BIGQUERY_POLICY_TAG') @use_profile('bigquery') def test__bigquery_update_column_policy_tag(self): From 007cb4516a075652186b72f5915fc62440b075d1 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Wed, 29 Jul 2020 17:19:54 -0400 Subject: [PATCH 7/8] Addressed PR comments --- core/dbt/contracts/graph/parsed.py | 8 +++-- core/dbt/contracts/graph/unparsed.py | 34 ++++++++++++++++++- core/dbt/parser/schemas.py | 1 + .../bigquery/dbt/adapters/bigquery/impl.py | 8 ++--- .../test_bigquery_update_columns.py | 5 ++- .../policy_tag_table.yml | 2 +- 6 files changed, 47 insertions(+), 11 deletions(-) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index a8387f93aa6..65dab4d42bf 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -13,13 +13,15 @@ ) from hologram import JsonSchemaMixin +from hologram.helpers import ExtensibleJsonSchemaMixin from dbt.clients.system import write_file from dbt.contracts.graph.unparsed import ( UnparsedNode, UnparsedDocumentation, Quoting, Docs, UnparsedBaseNode, FreshnessThreshold, ExternalTable, HasYamlMetadata, MacroArgument, UnparsedSourceDefinition, - UnparsedSourceTableDefinition, UnparsedColumn, TestDef + UnparsedSourceTableDefinition, UnparsedColumn, TestDef, + AdditionalPropertiesMixin ) from dbt.contracts.util import Replaceable from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -44,12 +46,14 @@ @dataclass -class ColumnInfo(JsonSchemaMixin, Replaceable): +class ColumnInfo(AdditionalPropertiesMixin, ExtensibleJsonSchemaMixin, + Replaceable): name: str description: str = '' meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None tags: List[str] = field(default_factory=list) + _extra: Dict[str, Any] = field(default_factory=dict) @dataclass diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 8810bf04b86..9d4ef4c5377 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -70,13 +70,45 @@ class Docs(JsonSchemaMixin, Replaceable): show: bool = True +class AdditionalPropertiesMixin: + """Make this class an extensible property. + + The underlying class definition must include a type definition for a field + named '_extra' that is of type `Dict[str, Any]`. + """ + @classmethod + def from_dict(cls, data, validate=True): + self = super().from_dict(data=data, validate=validate) + keys = self.to_dict(validate=False, omit_none=False) + for key, value in data.items(): + if key not in keys: + self.extra[key] = value + return self + + def to_dict(self, omit_none=True, validate=False): + data = super().to_dict(omit_none=omit_none, validate=validate) + data.update(self.extra) + return data + + def replace(self, **kwargs): + dct = self.to_dict(omit_none=False, validate=False) + dct.update(kwargs) + return self.from_dict(dct) + + @property + def extra(self): + return self._extra + + @dataclass -class HasDocs(JsonSchemaMixin, Replaceable): +class HasDocs(AdditionalPropertiesMixin, ExtensibleJsonSchemaMixin, + Replaceable): name: str description: str = '' meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None docs: Docs = field(default_factory=Docs) + _extra: Dict[str, Any] = field(default_factory=dict) TestDef = Union[Dict[str, Any], str] diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 87522b44d71..6c79034831a 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -100,6 +100,7 @@ def add( data_type=data_type, meta=meta, tags=tags, + _extra=column.extra ) @classmethod diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index 67357ad5e28..36049a46d23 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -612,10 +612,10 @@ def _update_column_dict(self, bq_column_dict, dbt_columns, parent=''): if dotted_column_name in dbt_columns: column_config = dbt_columns[dotted_column_name] bq_column_dict['description'] = column_config.get('description') - if column_config.get('tags'): - bq_column_dict['policyTags'] = {} - bq_column_dict['policyTags']['names'] = \ - column_config.get('tags') + if column_config.get('policy_tags'): + bq_column_dict['policyTags'] = { + 'names': column_config.get('policy_tags') + } new_fields = [] for child_col_dict in bq_column_dict.get('fields', list()): diff --git a/test/integration/022_bigquery_test/test_bigquery_update_columns.py b/test/integration/022_bigquery_test/test_bigquery_update_columns.py index 9ee007666b0..93179a19dc9 100644 --- a/test/integration/022_bigquery_test/test_bigquery_update_columns.py +++ b/test/integration/022_bigquery_test/test_bigquery_update_columns.py @@ -41,9 +41,8 @@ def test__bigquery_update_column_policy_tag(self): ) for schema_field in table.schema: - if self.policy_tag: - self.assertEquals(schema_field.policy_tags.names, - (self.policy_tag,)) + self.assertEquals(schema_field.policy_tags.names, + (self.policy_tag,)) class TestBigqueryUpdateColumnDescription(DBTIntegrationTest): diff --git a/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml b/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml index 649fe6f14cf..01a76c50dde 100644 --- a/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml +++ b/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml @@ -4,5 +4,5 @@ models: - name: policy_tag_table columns: - name: field - tags: + policy_tags: - '{{ var("policy_tag") }}' From d2c5783e5a1da32e15817e870e7af17ff22268e7 Mon Sep 17 00:00:00 2001 From: Azhar Dewji Date: Thu, 30 Jul 2020 11:02:23 -0400 Subject: [PATCH 8/8] Addressed PR commments --- CHANGELOG.md | 5 +++-- core/dbt/contracts/graph/parsed.py | 5 ++--- core/dbt/contracts/graph/unparsed.py | 30 ---------------------------- 3 files changed, 5 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 518d1c0e478..4ec9a43765b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Added new node selector methods (`config`, `test_type`, `test_name`, `package`) ([#2425](https://github.com/fishtown-analytics/dbt/issues/2425), [#2629](https://github.com/fishtown-analytics/dbt/pull/2629)) - Added option to specify profile when connecting to Redshift via IAM ([#2437](https://github.com/fishtown-analytics/dbt/issues/2437), [#2581](https://github.com/fishtown-analytics/dbt/pull/2581)) - Add more helpful error message for misconfiguration in profiles.yml ([#2569](https://github.com/fishtown-analytics/dbt/issues/2569), [#2627](https://github.com/fishtown-analytics/dbt/pull/2627)) +- Added support for setting policy tags for BigQuery columns ([#2586](https://github.com/fishtown-analytics/dbt/issues/2586), [#2589](https://github.com/fishtown-analytics/dbt/pull/2589)) ### Fixes - Adapter plugins can once again override plugins defined in core ([#2548](https://github.com/fishtown-analytics/dbt/issues/2548), [#2590](https://github.com/fishtown-analytics/dbt/pull/2590)) - Added `--selector` argument and support for `selectors.yml` file to define selection mechanisms. ([#2172](https://github.com/fishtown-analytics/dbt/issues/2172), [#2640](https://github.com/fishtown-analytics/dbt/pull/2640)) @@ -17,6 +18,7 @@ Contributors: - [@brunomurino](https://github.com/brunomurino) ([#2437](https://github.com/fishtown-analytics/dbt/pull/2581)) - [@DrMcTaco](https://github.com/DrMcTaco) ([#1030](https://github.com/fishtown-analytics/dbt/issues/1030)),[#2555](https://github.com/fishtown-analytics/dbt/pull/2555/)) - [@kning](https://github.com/kning) ([#2627](https://github.com/fishtown-analytics/dbt/pull/2627)) +- [@azhard](https://github.com/azhard) ([#2588](https://github.com/fishtown-analytics/dbt/pull/2588)) ## dbt 0.18.0b1 (June 08, 2020) @@ -32,7 +34,6 @@ Contributors: - Added support for BigQuery authorized views ([#1718](https://github.com/fishtown-analytics/dbt/issues/1718), [#2517](https://github.com/fishtown-analytics/dbt/pull/2517)) - Added support for altering BigQuery column types ([#2546](https://github.com/fishtown-analytics/dbt/issues/2546), [#2547](https://github.com/fishtown-analytics/dbt/pull/2547)) - Include row counts and bytes processed in log output for all BigQuery statement types ([#2526](https://github.com/fishtown-analytics/dbt/issues/2526)) -- Added support for setting policy tags for BigQuery columns ([#2586](https://github.com/fishtown-analytics/dbt/issues/2586), [#2589](https://github.com/fishtown-analytics/dbt/pull/2589)) ### Fixes @@ -45,7 +46,7 @@ Contributors: - [@alf-mindshift](https://github.com/alf-mindshift) ([#2431](https://github.com/fishtown-analytics/dbt/pull/2431)) - [@scarrucciu](https://github.com/scarrucciu) ([#2508](https://github.com/fishtown-analytics/dbt/pull/2508)) - [@southpolemonkey](https://github.com/southpolemonkey) ([#2511](https://github.com/fishtown-analytics/dbt/issues/2511)) - - [@azhard](https://github.com/azhard) ([#2517](https://github.com/fishtown-analytics/dbt/pull/2517), ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521)), [#2547](https://github.com/fishtown-analytics/dbt/pull/2547), [#2588](https://github.com/fishtown-analytics/dbt/pull/2588)) + - [@azhard](https://github.com/azhard) ([#2517](https://github.com/fishtown-analytics/dbt/pull/2517), ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521)), [#2547](https://github.com/fishtown-analytics/dbt/pull/2547)) - [@alepuccetti](https://github.com/alepuccetti) ([#2526](https://github.com/fishtown-analytics/dbt/issues/2526)) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 65dab4d42bf..9ac99c84400 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -20,10 +20,9 @@ UnparsedNode, UnparsedDocumentation, Quoting, Docs, UnparsedBaseNode, FreshnessThreshold, ExternalTable, HasYamlMetadata, MacroArgument, UnparsedSourceDefinition, - UnparsedSourceTableDefinition, UnparsedColumn, TestDef, - AdditionalPropertiesMixin + UnparsedSourceTableDefinition, UnparsedColumn, TestDef ) -from dbt.contracts.util import Replaceable +from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin from dbt.logger import GLOBAL_LOGGER as logger # noqa from dbt import flags from dbt.node_types import NodeType diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 9d4ef4c5377..105b41a2712 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -70,36 +70,6 @@ class Docs(JsonSchemaMixin, Replaceable): show: bool = True -class AdditionalPropertiesMixin: - """Make this class an extensible property. - - The underlying class definition must include a type definition for a field - named '_extra' that is of type `Dict[str, Any]`. - """ - @classmethod - def from_dict(cls, data, validate=True): - self = super().from_dict(data=data, validate=validate) - keys = self.to_dict(validate=False, omit_none=False) - for key, value in data.items(): - if key not in keys: - self.extra[key] = value - return self - - def to_dict(self, omit_none=True, validate=False): - data = super().to_dict(omit_none=omit_none, validate=validate) - data.update(self.extra) - return data - - def replace(self, **kwargs): - dct = self.to_dict(omit_none=False, validate=False) - dct.update(kwargs) - return self.from_dict(dct) - - @property - def extra(self): - return self._extra - - @dataclass class HasDocs(AdditionalPropertiesMixin, ExtensibleJsonSchemaMixin, Replaceable):