diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dccb571b..e5cab993e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ -## dbt-bigquery 1.2.0rc1 (Release TBD) +## dbt-bigquery 1.2.0rc1 (June 11, 2022) + +### Features +- Add grants to materializations ([#198](https://github.com/dbt-labs/dbt-bigquery/issues/198), [#212](https://github.com/dbt-labs/dbt-bigquery/pull/212)) ### Under the hood - Modify `BigQueryColumn.numeric_type` to always exclude precision + scale, since the functionality of ["parametrized data types on BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#parameterized_data_types) is highly constrained ([#214](https://github.com/dbt-labs/dbt-bigquery/pull/214)) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 344ffb22e..7c66b014d 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -757,6 +757,14 @@ def grant_access_to(self, entity, entity_type, role, grant_target_dict): dataset.access_entries = access_entries client.update_dataset(dataset, ["access_entries"]) + @available.parse_none + def get_dataset_location(self, relation): + conn = self.connections.get_thread_connection() + client = conn.handle + dataset_ref = self.connections.dataset_ref(relation.project, relation.dataset) + dataset = client.get_dataset(dataset_ref) + return dataset.location + def get_rows_different_sql( # type: ignore[override] self, relation_a: BigQueryRelation, diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 8156e360d..7224de8cf 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -1,7 +1,10 @@ from dataclasses import dataclass from typing import Optional +from itertools import chain, islice + from dbt.adapters.base.relation import BaseRelation, ComponentName, InformationSchema +from dbt.exceptions import raise_compiler_error from dbt.utils import filter_null_values from typing import TypeVar @@ -12,6 +15,7 @@ @dataclass(frozen=True, eq=False, repr=False) class BigQueryRelation(BaseRelation): quote_character: str = "`" + location: Optional[str] = None def matches( self, @@ -52,6 +56,7 @@ def information_schema(self, identifier: Optional[str] = None) -> "BigQueryInfor @dataclass(frozen=True, eq=False, repr=False) class BigQueryInformationSchema(InformationSchema): quote_character: str = "`" + location: Optional[str] = None @classmethod def get_include_policy(cls, relation, information_schema_view): @@ -63,11 +68,49 @@ def get_include_policy(cls, relation, information_schema_view): if information_schema_view == "__TABLES__": identifier = False + # In the future, let's refactor so that location/region can also be a + # ComponentName, so that we can have logic like: + # + # region = False + # if information_schema_view == "OBJECT_PRIVILEGES": + # region = True + return relation.include_policy.replace( schema=schema, identifier=identifier, ) + def get_region_identifier(self) -> str: + region_id = f"region-{self.location}" + return self.quoted(region_id) + + @classmethod + def from_relation(cls, relation, information_schema_view): + info_schema = super().from_relation(relation, information_schema_view) + if information_schema_view == "OBJECT_PRIVILEGES": + # OBJECT_PRIVILEGES require a location. If the location is blank there is nothing + # the user can do about it. + if not relation.location: + msg = ( + f'No location/region found when trying to retrieve "{information_schema_view}"' + ) + raise raise_compiler_error(msg) + info_schema = info_schema.incorporate(location=relation.location) + return info_schema + + # override this method to interpolate the region identifier, + # if a location is required for this information schema view + def _render_iterator(self): + iterator = super()._render_iterator() + if self.location: + return chain( + islice(iterator, 1), # project, + [(None, self.get_region_identifier())], # region id, + islice(iterator, 1, None), # remaining components + ) + else: + return iterator + def replace(self, **kwargs): if "information_schema_view" in kwargs: view = kwargs["information_schema_view"] diff --git a/dbt/include/bigquery/macros/adapters/apply_grants.sql b/dbt/include/bigquery/macros/adapters/apply_grants.sql new file mode 100644 index 000000000..e344862ae --- /dev/null +++ b/dbt/include/bigquery/macros/adapters/apply_grants.sql @@ -0,0 +1,20 @@ +{% macro bigquery__get_show_grant_sql(relation) %} + {% set location = adapter.get_dataset_location(relation) %} + {% set relation = relation.incorporate(location=location) %} + + select privilege_type, grantee + from {{ relation.information_schema("OBJECT_PRIVILEGES") }} + where object_schema = "{{ relation.dataset }}" + and object_name = "{{ relation.identifier }}" + -- filter out current user + and split(grantee, ':')[offset(1)] != session_user() +{% endmacro %} + + +{%- macro bigquery__get_grant_sql(relation, privilege, grantee) -%} + grant `{{ privilege }}` on {{ relation.type }} {{ relation }} to {{ '\"' + grantee|join('\", \"') + '\"' }} +{%- endmacro -%} + +{%- macro bigquery__get_revoke_sql(relation, privilege, grantee) -%} + revoke `{{ privilege }}` on {{ relation.type }} {{ relation }} from {{ '\"' + grantee|join('\", \"') + '\"' }} +{%- endmacro -%} diff --git a/dbt/include/bigquery/macros/materializations/copy.sql b/dbt/include/bigquery/macros/materializations/copy.sql index 6a86fbe44..8285dc845 100644 --- a/dbt/include/bigquery/macros/materializations/copy.sql +++ b/dbt/include/bigquery/macros/materializations/copy.sql @@ -16,7 +16,7 @@ {{ source_array.append(source(*src_table)) }} {% endfor %} - {# Call adapter's copy_table function #} + {# Call adapter copy_table function #} {%- set result_str = adapter.copy_table( source_array, destination, @@ -26,6 +26,7 @@ {# Clean up #} {{ run_hooks(post_hooks) }} + {%- do apply_grants(target_relation, grant_config) -%} {{ adapter.commit() }} {{ return({'relations': [destination]}) }} diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index b6d387890..8cf2ab65c 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -152,6 +152,9 @@ {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + {{ run_hooks(pre_hooks) }} {% if existing_relation is none %} @@ -197,6 +200,9 @@ {% set target_relation = this.incorporate(type='table') %} + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke) %} + {% do persist_docs(target_relation, model) %} {{ return({'relations': [target_relation]}) }} diff --git a/dbt/include/bigquery/macros/materializations/table.sql b/dbt/include/bigquery/macros/materializations/table.sql index a7452265b..9e63637c1 100644 --- a/dbt/include/bigquery/macros/materializations/table.sql +++ b/dbt/include/bigquery/macros/materializations/table.sql @@ -5,6 +5,9 @@ {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} {%- set target_relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%} + -- grab current tables grants config for comparision later on + {%- set grant_config = config.get('grants') -%} + {{ run_hooks(pre_hooks) }} {# @@ -30,6 +33,9 @@ {{ run_hooks(post_hooks) }} + {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke) %} + {% do persist_docs(target_relation, model) %} {{ return({'relations': [target_relation]}) }} diff --git a/dbt/include/bigquery/macros/materializations/view.sql b/dbt/include/bigquery/macros/materializations/view.sql index 97e3d2761..e68a51421 100644 --- a/dbt/include/bigquery/macros/materializations/view.sql +++ b/dbt/include/bigquery/macros/materializations/view.sql @@ -9,9 +9,13 @@ {% materialization view, adapter='bigquery' -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + {% set to_return = create_or_replace_view() %} {% set target_relation = this.incorporate(type='view') %} + {% do persist_docs(target_relation, model) %} {% if config.get('grant_access_to') %} diff --git a/test.env.example b/test.env.example index 2065e4393..f2e59e6d0 100644 --- a/test.env.example +++ b/test.env.example @@ -1,3 +1,7 @@ BIGQUERY_TEST_ALT_DATABASE= BIGQUERY_TEST_NO_ACCESS_DATABASE= BIGQUERY_TEST_SERVICE_ACCOUNT_JSON='{}' + +DBT_TEST_USER_1="group:buildbot@dbtlabs.com" +DBT_TEST_USER_2="group:dev-core@dbtlabs.com" +DBT_TEST_USER_3="serviceAccount:dbt-integration-test-user@dbt-test-env.iam.gserviceaccount.com" diff --git a/tests/conftest.py b/tests/conftest.py index 4bbdb00e0..69a29b39c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,6 @@ def oauth_target(): 'type': 'bigquery', 'method': 'oauth', 'threads': 1, - # project isn't needed if you configure a default, via 'gcloud config set project' } diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py new file mode 100644 index 000000000..b35e4787e --- /dev/null +++ b/tests/functional/adapter/test_grants.py @@ -0,0 +1,44 @@ +import pytest + +from dbt.tests.adapter.grants.base_grants import BaseGrants +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants +from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants + + +class BaseGrantsBigQuery(BaseGrants): + def privilege_grantee_name_overrides(self): + return { + "select": "roles/bigquery.dataViewer", + "insert": "roles/bigquery.dataEditor", + "fake_privilege": "roles/invalid", + "invalid_user": "user:fake@dbtlabs.com", + } + +class TestModelGrantsBigQuery(BaseGrantsBigQuery, BaseModelGrants): + pass + + +class TestIncrementalGrantsBigQuery(BaseGrantsBigQuery, BaseIncrementalGrants): + pass + + +class TestSeedGrantsBigQuery(BaseGrantsBigQuery, BaseSeedGrants): + # seeds in dbt-bigquery are always "full refreshed," in such a way that + # the grants do not carry over + def seeds_support_partial_refresh(self): + return False + + +class TestSnapshotGrantsBigQuery(BaseGrantsBigQuery, BaseSnapshotGrants): + pass + + +class TestInvalidGrantsBigQuery(BaseGrantsBigQuery, BaseInvalidGrants): + def grantee_does_not_exist_error(self): + return "User fake@dbtlabs.com does not exist." + + def privilege_does_not_exist_error(self): + return "Role roles/invalid is not supported for this resource."